From 10299ad8df8b249dffb4aeb3a86b20a9c82634f3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 15 Jul 2026 23:24:31 +0800 Subject: [PATCH 01/34] [improvement](be) Add selection-aware Parquet decode kernel ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: FileScannerV2 currently materializes Parquet values through Arrow and repeatedly allocates conversion and nested-column scratch. Introduce an Arrow-independent physical column schema and selection-aware flat decode contract in the existing Doris Parquet kernel, batch string materialization, retain scalar and complex-reader scratch across batches, and document the target shared-level-plan interface for native complex decoding. The legacy v1 production call path remains unchanged. ### Release note None ### Check List (For Author) - Test: Unit Test - Remote ASAN BE build - 120 targeted Parquet BE unit tests - Behavior changed: No; this adds migration interfaces and internal allocation optimizations - Does this need documentation: Yes; updated the FileScannerV2 Parquet design document and format_v2 review guide --- .../parquet/byte_array_dict_decoder.cpp | 15 +- .../format/parquet/byte_array_dict_decoder.h | 3 + .../parquet/byte_array_plain_decoder.cpp | 25 +- .../format/parquet/byte_array_plain_decoder.h | 7 + .../format/parquet/delta_bit_pack_decoder.h | 19 +- be/src/format/parquet/parquet_common.cpp | 178 +++++++++++ be/src/format/parquet/parquet_common.h | 65 +++- .../parquet/vparquet_column_chunk_reader.cpp | 87 ++++- .../parquet/vparquet_column_chunk_reader.h | 55 +++- be/src/format_v2/AGENTS.md | 93 ++++++ .../parquet/reader/list_column_reader.cpp | 20 +- .../parquet/reader/list_column_reader.h | 4 + .../parquet/reader/map_column_reader.cpp | 26 +- .../parquet/reader/map_column_reader.h | 5 + .../parquet/reader/parquet_leaf_reader.cpp | 102 +++--- .../parquet/reader/parquet_leaf_reader.h | 35 ++ .../parquet/reader/scalar_column_reader.cpp | 53 ++-- .../parquet/reader/scalar_column_reader.h | 20 +- .../parquet/reader/struct_column_reader.cpp | 30 +- .../parquet/reader/struct_column_reader.h | 5 + be/src/format_v2/parquet/selection_vector.h | 19 +- .../parquet/byte_array_dict_decoder_test.cpp | 27 ++ .../parquet/byte_array_plain_decoder_test.cpp | 39 +++ .../byte_stream_split_decoder_test.cpp | 32 ++ .../parquet/delta_bit_pack_decoder_test.cpp | 26 ++ .../parquet/fix_length_plain_decoder_test.cpp | 29 ++ .../format/parquet/parquet_common_test.cpp | 216 +++++++++++++ .../format/parquet/parquet_thrift_test.cpp | 16 + .../parquet/parquet_reader_control_test.cpp | 82 ++++- docs/file-scanner-v2-parquet-scan-design.md | 300 +++++++++++++++++- 30 files changed, 1462 insertions(+), 171 deletions(-) diff --git a/be/src/format/parquet/byte_array_dict_decoder.cpp b/be/src/format/parquet/byte_array_dict_decoder.cpp index c7ca433223ed7a..1bc1a0b5b4acb5 100644 --- a/be/src/format/parquet/byte_array_dict_decoder.cpp +++ b/be/src/format/parquet/byte_array_dict_decoder.cpp @@ -145,22 +145,20 @@ Status ByteArrayDictDecoder::_decode_values(MutableColumnPtr& doris_column, Data } size_t dict_index = 0; + _selected_values.clear(); + _selected_values.reserve(select_vector.num_values() - select_vector.num_filtered()); ColumnSelectVector::DataReadType read_type; while (size_t run_length = select_vector.get_next_run(&read_type)) { switch (read_type) { case ColumnSelectVector::CONTENT: { - DorisVector string_values; - string_values.reserve(run_length); for (size_t i = 0; i < run_length; ++i) { - string_values.emplace_back(_dict_items[_indexes[dict_index++]]); + _selected_values.emplace_back(_dict_items[_indexes[dict_index++]]); } - doris_column->insert_many_strings_overflow(string_values.data(), run_length, - _max_value_length); break; } case ColumnSelectVector::NULL_DATA: { - doris_column->insert_many_defaults(run_length); + _selected_values.insert(_selected_values.end(), run_length, StringRef("", 0)); break; } case ColumnSelectVector::FILTERED_CONTENT: { @@ -173,6 +171,11 @@ Status ByteArrayDictDecoder::_decode_values(MutableColumnPtr& doris_column, Data } } } + DCHECK_EQ(_selected_values.size(), select_vector.num_values() - select_vector.num_filtered()); + if (!_selected_values.empty()) { + doris_column->insert_many_strings_overflow(_selected_values.data(), _selected_values.size(), + _max_value_length); + } return Status::OK(); } diff --git a/be/src/format/parquet/byte_array_dict_decoder.h b/be/src/format/parquet/byte_array_dict_decoder.h index ff79b4ba94fe83..91fbc14cd1f161 100644 --- a/be/src/format/parquet/byte_array_dict_decoder.h +++ b/be/src/format/parquet/byte_array_dict_decoder.h @@ -60,6 +60,9 @@ class ByteArrayDictDecoder final : public BaseDictDecoder { // For dictionary encoding DorisVector _dict_items; DorisVector _dict_data; + // Selected dictionary values are gathered once and appended in one ColumnString resize/copy. + // The capacity is retained across pages and batches. + DorisVector _selected_values; size_t _max_value_length; }; diff --git a/be/src/format/parquet/byte_array_plain_decoder.cpp b/be/src/format/parquet/byte_array_plain_decoder.cpp index d9048c5549cc44..343518b5d23264 100644 --- a/be/src/format/parquet/byte_array_plain_decoder.cpp +++ b/be/src/format/parquet/byte_array_plain_decoder.cpp @@ -18,7 +18,6 @@ #include "format/parquet/byte_array_plain_decoder.h" #include -#include #include "core/column/column.h" #include "core/data_type/data_type_nullable.h" @@ -62,37 +61,29 @@ template Status ByteArrayPlainDecoder::_decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector, bool is_dict_filter) { + _selected_values.clear(); + _selected_values.reserve(select_vector.num_values() - select_vector.num_filtered()); ColumnSelectVector::DataReadType read_type; while (size_t run_length = select_vector.get_next_run(&read_type)) { switch (read_type) { case ColumnSelectVector::CONTENT: { - std::vector string_values; - string_values.reserve(run_length); for (size_t i = 0; i < run_length; ++i) { uint32_t length = 0; RETURN_IF_ERROR(read_length(_data, &_offset, &length)); if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { return Status::IOError("Can't read enough bytes in plain decoder"); } - string_values.emplace_back(_data->data + _offset, length); + _selected_values.emplace_back(_data->data + _offset, length); _offset += length; } - doris_column->insert_many_strings(&string_values[0], run_length); break; } case ColumnSelectVector::NULL_DATA: { - doris_column->insert_many_defaults(run_length); + _selected_values.insert(_selected_values.end(), run_length, StringRef("", 0)); break; } case ColumnSelectVector::FILTERED_CONTENT: { - for (int i = 0; i < run_length; ++i) { - uint32_t length = 0; - RETURN_IF_ERROR(read_length(_data, &_offset, &length)); - if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { - return Status::IOError("Can't read enough bytes in plain decoder"); - } - _offset += length; - } + RETURN_IF_ERROR(skip_values(run_length)); break; } case ColumnSelectVector::FILTERED_NULL: { @@ -101,6 +92,12 @@ Status ByteArrayPlainDecoder::_decode_values(MutableColumnPtr& doris_column, Dat } } } + DCHECK_EQ(_selected_values.size(), select_vector.num_values() - select_vector.num_filtered()); + if (!_selected_values.empty()) { + // ColumnString calculates the aggregate byte length before copying, so this call grows the + // chars buffer and offsets exactly once for the whole selected batch. + doris_column->insert_many_strings(_selected_values.data(), _selected_values.size()); + } return Status::OK(); } diff --git a/be/src/format/parquet/byte_array_plain_decoder.h b/be/src/format/parquet/byte_array_plain_decoder.h index 192824a2aab7a3..fd3f7d3fef52b5 100644 --- a/be/src/format/parquet/byte_array_plain_decoder.h +++ b/be/src/format/parquet/byte_array_plain_decoder.h @@ -21,10 +21,12 @@ #include #include +#include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" #include "core/data_type/data_type.h" +#include "core/string_ref.h" #include "core/types.h" #include "format/format_common.h" #include "format/parquet/decoder.h" @@ -52,6 +54,11 @@ class ByteArrayPlainDecoder final : public Decoder { ColumnSelectVector& select_vector, bool is_dict_filter); Status skip_values(size_t num_values) override; + +private: + // References point into the current page buffer and are valid until the batch is appended. + // Keeping the vector on the decoder reuses its capacity across pages and batches. + std::vector _selected_values; }; } // namespace doris diff --git a/be/src/format/parquet/delta_bit_pack_decoder.h b/be/src/format/parquet/delta_bit_pack_decoder.h index 6257e4f214a182..e32cadb86be941 100644 --- a/be/src/format/parquet/delta_bit_pack_decoder.h +++ b/be/src/format/parquet/delta_bit_pack_decoder.h @@ -50,23 +50,23 @@ class DeltaDecoder : public Decoder { template Status decode_byte_array(const std::vector& decoded_vals, MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector) { + _selected_string_values.clear(); + _selected_string_values.reserve(select_vector.num_values() - select_vector.num_filtered()); ColumnSelectVector::DataReadType read_type; int value_idx = 0; while (size_t run_length = select_vector.get_next_run(&read_type)) { switch (read_type) { case ColumnSelectVector::CONTENT: { - std::vector string_values; - string_values.reserve(run_length); for (size_t i = 0; i < run_length; ++i) { size_t length = decoded_vals[value_idx].size; - string_values.emplace_back(decoded_vals[value_idx].data, length); + _selected_string_values.emplace_back(decoded_vals[value_idx].data, length); value_idx++; } - doris_column->insert_many_strings(&string_values[0], run_length); break; } case ColumnSelectVector::NULL_DATA: { - doris_column->insert_many_defaults(run_length); + _selected_string_values.insert(_selected_string_values.end(), run_length, + StringRef("", 0)); break; } case ColumnSelectVector::FILTERED_CONTENT: { @@ -79,6 +79,12 @@ class DeltaDecoder : public Decoder { } } } + DCHECK_EQ(_selected_string_values.size(), + select_vector.num_values() - select_vector.num_filtered()); + if (!_selected_string_values.empty()) { + doris_column->insert_many_strings(_selected_string_values.data(), + _selected_string_values.size()); + } return Status::OK(); } @@ -138,6 +144,9 @@ class DeltaDecoder : public Decoder { } // Convert decoded value to doris type value. std::unique_ptr _type_converted_decoder; + // Values reference decoder-owned page scratch. Retaining this vector avoids one allocation per + // CONTENT run and lets ColumnString resize once for the whole selected batch. + std::vector _selected_string_values; }; /** diff --git a/be/src/format/parquet/parquet_common.cpp b/be/src/format/parquet/parquet_common.cpp index 26ec8583b7753a..743f4b393154f4 100644 --- a/be/src/format/parquet/parquet_common.cpp +++ b/be/src/format/parquet/parquet_common.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/cast_set.h" #include "core/types.h" #include "util/simd/bits.h" @@ -120,6 +122,10 @@ Status FilterMap::generate_nested_filter_map(const std::vector& rep_lev Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, const std::unordered_set* skipped_indices) { + _uses_index_selection = false; + _selection_indices = nullptr; + _selected_count = 0; + _run_length_null_map = &run_length_null_map; _num_values = num_values; _num_nulls = 0; _read_index = 0; @@ -218,6 +224,178 @@ Status ColumnSelectVector::init(const std::vector& run_length_null_map return Status::OK(); } +Status ColumnSelectVector::_validate_null_runs(const std::vector& run_length_null_map, + size_t num_values) const { + if (run_length_null_map.empty()) { + return Status::OK(); + } + + size_t run_length_sum = 0; + for (const auto run_length : run_length_null_map) { + run_length_sum += run_length; + if (run_length_sum > num_values) { + return Status::InvalidArgument( + fmt::format("Parquet null runs describe {} rows, which exceeds num_values {}", + run_length_sum, num_values)); + } + } + if (run_length_sum != num_values) { + return Status::InvalidArgument(fmt::format( + "Parquet null runs describe {} rows, expected {}", run_length_sum, num_values)); + } + return Status::OK(); +} + +Status ColumnSelectVector::init_from_selection(const std::vector& run_length_null_map, + size_t num_values, NullMap* const null_map, + const uint16_t* selected_indices, + size_t selected_count) { + RETURN_IF_ERROR(_validate_null_runs(run_length_null_map, num_values)); + if (selected_count > num_values) { + return Status::InvalidArgument( + fmt::format("Parquet selection contains {} rows, which exceeds num_values {}", + selected_count, num_values)); + } + if (selected_indices == nullptr && selected_count != 0 && selected_count != num_values) { + return Status::InvalidArgument( + "A partial Parquet selection must provide explicit row indices"); + } + if (selected_indices != nullptr) { + for (size_t i = 0; i < selected_count; ++i) { + if (selected_indices[i] >= num_values) { + return Status::InvalidArgument( + fmt::format("Parquet selection index {} is outside [0, {})", + selected_indices[i], num_values)); + } + if (i != 0 && selected_indices[i - 1] >= selected_indices[i]) { + return Status::InvalidArgument( + "Parquet selection indices must be strictly increasing"); + } + } + } + + _data_map.clear(); + _run_length_null_map = &run_length_null_map; + _selection_indices = selected_indices; + _selected_count = selected_count; + _num_values = num_values; + _num_filtered = num_values - selected_count; + _has_filter = selected_count != num_values; + _uses_index_selection = _has_filter; + _read_index = 0; + _num_nulls = 0; + + bool is_null = false; + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + + if (null_map != nullptr && selected_count != 0) { + const size_t null_map_offset = null_map->size(); + null_map->resize(null_map_offset + selected_count); + if (run_length_null_map.empty() || _num_nulls == 0) { + memset(null_map->data() + null_map_offset, 0, selected_count); + } else if (_num_nulls == num_values) { + memset(null_map->data() + null_map_offset, 1, selected_count); + } else { + size_t run_index = 0; + size_t run_end = run_length_null_map[0]; + bool selected_row_is_null = false; + for (size_t i = 0; i < selected_count; ++i) { + const size_t selected_row = selected_indices == nullptr ? i : selected_indices[i]; + while (selected_row >= run_end) { + ++run_index; + selected_row_is_null = !selected_row_is_null; + run_end += run_length_null_map[run_index]; + } + (*null_map)[null_map_offset + i] = selected_row_is_null; + } + } + } + + _reset_selection_cursor(); + return Status::OK(); +} + +void ColumnSelectVector::_reset_selection_cursor() { + _selected_index = 0; + _selection_row_index = 0; + _selection_null_run_index = 0; + _selection_row_is_null = false; + + if (_run_length_null_map->empty()) { + _selection_null_run_remaining = _num_values; + return; + } + + _selection_null_run_remaining = (*_run_length_null_map)[0]; + while (_selection_null_run_remaining == 0 && + _selection_null_run_index + 1 < _run_length_null_map->size()) { + ++_selection_null_run_index; + _selection_row_is_null = !_selection_row_is_null; + _selection_null_run_remaining = (*_run_length_null_map)[_selection_null_run_index]; + } +} + +ColumnSelectVector::DataReadType ColumnSelectVector::_current_selection_type() const { + const bool selected = _selected_index < _selected_count && + _selection_indices[_selected_index] == _selection_row_index; + if (selected) { + return _selection_row_is_null ? NULL_DATA : CONTENT; + } + return _selection_row_is_null ? FILTERED_NULL : FILTERED_CONTENT; +} + +void ColumnSelectVector::_advance_selection_cursor(size_t run_length) { + DCHECK_GT(run_length, 0); + DCHECK_LE(run_length, _selection_null_run_remaining); + const size_t next_row_index = _selection_row_index + run_length; + while (_selected_index < _selected_count && + _selection_indices[_selected_index] < next_row_index) { + ++_selected_index; + } + _selection_row_index = next_row_index; + _selection_null_run_remaining -= run_length; + + while (_selection_row_index < _num_values && _selection_null_run_remaining == 0) { + ++_selection_null_run_index; + _selection_row_is_null = !_selection_row_is_null; + _selection_null_run_remaining = (*_run_length_null_map)[_selection_null_run_index]; + } +} + +size_t ColumnSelectVector::_get_next_selection_run(DataReadType* data_read_type) { + if (_selection_row_index == _num_values) { + return 0; + } + + *data_read_type = _current_selection_type(); + size_t run_length = 0; + if (*data_read_type == CONTENT || *data_read_type == NULL_DATA) { + const size_t null_run_end = _selection_row_index + _selection_null_run_remaining; + size_t selected_index = _selected_index; + size_t expected_row = _selection_row_index; + while (selected_index < _selected_count && + _selection_indices[selected_index] == expected_row && expected_row < null_run_end) { + ++selected_index; + ++expected_row; + } + run_length = selected_index - _selected_index; + } else { + const size_t next_selected_row = _selected_index == _selected_count + ? _num_values + : _selection_indices[_selected_index]; + run_length = + std::min(_selection_null_run_remaining, next_selected_row - _selection_row_index); + } + DCHECK_GT(run_length, 0); + _advance_selection_cursor(run_length); + return run_length; +} + ParsedVersion::ParsedVersion(std::string application, std::optional version, std::optional app_build_hash) : _application(std::move(application)), diff --git a/be/src/format/parquet/parquet_common.h b/be/src/format/parquet/parquet_common.h index 277979932ff2d3..994b2723aefde5 100644 --- a/be/src/format/parquet/parquet_common.h +++ b/be/src/format/parquet/parquet_common.h @@ -93,6 +93,29 @@ class ColumnSelectVector { NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, const std::unordered_set* skipped_indices = nullptr); + /** + * Build a decoder plan from sorted logical-row indices. + * + * This is the native late-materialization contract used by the new Parquet reader. The + * selection indices address the logical rows described by `run_length_null_map`, including + * null rows. They must be strictly increasing and smaller than `num_values`. A null pointer is + * the dense identity selection and therefore requires `selected_count == num_values`; it is + * also accepted for an empty selection. + * + * `run_length_null_map` alternates non-null and null run lengths, starting with a non-null run. + * Zero-length runs are valid because they preserve the alternation. An empty vector means that + * all logical rows are non-null. When `null_map` is non-null, this method appends exactly + * `selected_count` entries in output order. + * + * The plan does not copy `selected_indices` or `run_length_null_map`. Both must remain alive + * and unchanged until the decoder has consumed all runs through get_next_run(). The plan owns + * no per-row action array: it merges the two sorted streams while the decoder advances, so a + * fragmented selection does not allocate O(num_values) scratch memory. + */ + Status init_from_selection(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, const uint16_t* selected_indices, + size_t selected_count); + size_t num_values() const { return _num_values; } size_t num_nulls() const { return _num_nulls; } @@ -105,6 +128,9 @@ class ColumnSelectVector { size_t get_next_run(DataReadType* data_read_type) { DCHECK_EQ(_has_filter, has_filter); if constexpr (has_filter) { + if (_uses_index_selection) { + return _get_next_selection_run(data_read_type); + } if (_read_index == _num_values) { return 0; } @@ -121,6 +147,14 @@ class ColumnSelectVector { *data_read_type = type; return run_length; } else { + if (_run_length_null_map->empty()) { + if (_read_index != 0) { + return 0; + } + ++_read_index; + *data_read_type = CONTENT; + return _num_values; + } size_t run_length = 0; while (run_length == 0) { if (_read_index == (*_run_length_null_map).size()) { @@ -134,14 +168,33 @@ class ColumnSelectVector { } private: + Status _validate_null_runs(const std::vector& run_length_null_map, + size_t num_values) const; + void _reset_selection_cursor(); + DataReadType _current_selection_type() const; + void _advance_selection_cursor(size_t run_length); + size_t _get_next_selection_run(DataReadType* data_read_type); + std::vector _data_map; // the length of non-null values and null values are arranged in turn. - const std::vector* _run_length_null_map; - bool _has_filter; - size_t _num_values; - size_t _num_nulls; - size_t _num_filtered; - size_t _read_index; + const std::vector* _run_length_null_map = nullptr; + bool _has_filter = false; + size_t _num_values = 0; + size_t _num_nulls = 0; + size_t _num_filtered = 0; + size_t _read_index = 0; + + // INDEX selection mode. The input arrays are borrowed for the duration of one decode call. + // `_selection_row_index` is the logical row (nulls included); decoder payload cursors advance + // only for CONTENT and FILTERED_CONTENT runs. + const uint16_t* _selection_indices = nullptr; + bool _uses_index_selection = false; + size_t _selected_count = 0; + size_t _selected_index = 0; + size_t _selection_row_index = 0; + size_t _selection_null_run_index = 0; + size_t _selection_null_run_remaining = 0; + bool _selection_row_is_null = false; }; enum class ColumnOrderName { UNDEFINED, TYPE_DEFINED_ORDER }; diff --git a/be/src/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/format/parquet/vparquet_column_chunk_reader.cpp index b4b919f187073c..722e6613bc5176 100644 --- a/be/src/format/parquet/vparquet_column_chunk_reader.cpp +++ b/be/src/format/parquet/vparquet_column_chunk_reader.cpp @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include #include @@ -49,14 +51,33 @@ struct IOContext; } // namespace doris namespace doris { +ParquetColumnChunkSchema ParquetColumnChunkSchema::from_field_schema( + const FieldSchema& field_schema) { + return ParquetColumnChunkSchema(field_schema.physical_type, + field_schema.parquet_schema.__isset.type_length + ? field_schema.parquet_schema.type_length + : -1, + field_schema.definition_level, field_schema.repetition_level, + field_schema.repeated_parent_def_level); +} + template ColumnChunkReader::ColumnChunkReader( io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx) - : _field_schema(field_schema), - _max_rep_level(field_schema->repetition_level), - _max_def_level(field_schema->definition_level), + : ColumnChunkReader(reader, column_chunk, + ParquetColumnChunkSchema::from_field_schema(*field_schema), + offset_index, total_rows, io_ctx, page_read_ctx) {} + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + ParquetColumnChunkSchema chunk_schema, const tparquet::OffsetIndex* offset_index, + size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx) + : _chunk_schema(chunk_schema), + _max_rep_level(chunk_schema.max_repetition_level), + _max_def_level(chunk_schema.max_definition_level), _stream_reader(reader), _metadata(column_chunk->meta_data), _offset_index(offset_index), @@ -88,9 +109,9 @@ Status ColumnChunkReader::skip_nested_values( for (size_t idx = 0; idx < def_levels.size(); idx++) { level_t def_level = def_levels[idx]; - if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + if (IN_COLLECTION && def_level < _chunk_schema.repeated_parent_definition_level) { no_value_cnt++; - } else if (def_level < _field_schema->definition_level) { + } else if (def_level < _chunk_schema.max_definition_level) { no_value_cnt++; } else { value_cnt++; @@ -102,6 +123,58 @@ Status ColumnChunkReader::skip_nested_values( return Status::OK(); } +template +Status ColumnChunkReader::decode_flat_values( + MutableColumnPtr& doris_column, DataTypePtr& data_type, size_t num_values, + const uint16_t* selected_indices, size_t selected_count, NullMap* const null_map, + bool is_dict_filter) { + DCHECK(!IN_COLLECTION); + DCHECK_EQ(_max_rep_level, 0); + + _definition_runs.clear(); + if (_max_def_level != 0) { + // The first run is always non-null. A leading zero preserves parity when the first decoded + // level is null. Two zero runs preserve parity when one logical run exceeds uint16_t. + _definition_runs.emplace_back(0); + bool current_run_is_null = false; + size_t levels_read = 0; + while (levels_read < num_values) { + level_t definition_level = -1; + const size_t run_length = + _def_level_decoder.get_next_run(&definition_level, num_values - levels_read); + if (run_length == 0) { + return Status::Corruption( + "Parquet definition-level stream ended after {} of {} flat values", + levels_read, num_values); + } + + const bool run_is_null = definition_level < _max_def_level; + if (run_is_null != current_run_is_null) { + _definition_runs.emplace_back(0); + current_run_is_null = run_is_null; + } + size_t remaining = run_length; + while (remaining != 0) { + const size_t available = + std::numeric_limits::max() - _definition_runs.back(); + const size_t appended = std::min(remaining, available); + _definition_runs.back() += static_cast(appended); + remaining -= appended; + if (remaining != 0) { + _definition_runs.emplace_back(0); + _definition_runs.emplace_back(0); + } + } + levels_read += run_length; + } + } + + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init_from_selection(_definition_runs, num_values, null_map, + selected_indices, selected_count)); + return decode_values(doris_column, data_type, select_vector, is_dict_filter); +} + template Status ColumnChunkReader::_parse_first_page_header() { RETURN_IF_ERROR(parse_page_header()); @@ -711,7 +784,7 @@ Status ColumnChunkReader::load_cross_page_nested_ro template int32_t ColumnChunkReader::_get_type_length() { - switch (_field_schema->physical_type) { + switch (_chunk_schema.physical_type) { case tparquet::Type::INT32: [[fallthrough]]; case tparquet::Type::FLOAT: @@ -723,7 +796,7 @@ int32_t ColumnChunkReader::_get_type_length() { case tparquet::Type::INT96: return 12; case tparquet::Type::FIXED_LEN_BYTE_ARRAY: - return _field_schema->parquet_schema.type_length; + return _chunk_schema.type_length; default: return -1; } diff --git a/be/src/format/parquet/vparquet_column_chunk_reader.h b/be/src/format/parquet/vparquet_column_chunk_reader.h index b117f6c6652e7e..31629a6f05f73a 100644 --- a/be/src/format/parquet/vparquet_column_chunk_reader.h +++ b/be/src/format/parquet/vparquet_column_chunk_reader.h @@ -70,6 +70,34 @@ struct ColumnChunkReaderStatistics { int64_t page_cache_decompressed_hit_counter = 0; }; +/** + * Immutable physical schema needed by the page/encoding kernel for one leaf column. + * + * Keep this contract independent of both Arrow's ColumnDescriptor and Doris' legacy + * FieldSchema. A native v2 reader can build it directly from the Thrift footer, while the v1 + * reader uses from_field_schema() during migration. Logical-type conversion belongs above this + * layer: the chunk reader only needs the physical carrier, fixed width, and Dremel thresholds to + * decide which payload values exist. + */ +struct ParquetColumnChunkSchema { + ParquetColumnChunkSchema(tparquet::Type::type physical_type, int32_t type_length, + level_t max_definition_level, level_t max_repetition_level, + level_t repeated_parent_definition_level) + : physical_type(physical_type), + type_length(type_length), + max_definition_level(max_definition_level), + max_repetition_level(max_repetition_level), + repeated_parent_definition_level(repeated_parent_definition_level) {} + + const tparquet::Type::type physical_type; + const int32_t type_length; + const level_t max_definition_level; + const level_t max_repetition_level; + const level_t repeated_parent_definition_level; + + static ParquetColumnChunkSchema from_field_schema(const FieldSchema& field_schema); +}; + /** * Read and decode parquet column data into doris block column. *

Usage:

@@ -97,6 +125,10 @@ class ColumnChunkReader { FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_row, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx); + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + ParquetColumnChunkSchema chunk_schema, + const tparquet::OffsetIndex* offset_index, size_t total_row, + io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx); ~ColumnChunkReader() = default; // Initialize chunk reader, will generate the decoder and codec. @@ -133,6 +165,25 @@ class ColumnChunkReader { Status decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector, bool is_dict_filter); + /** + * Decode selected rows from the current flat data-page slice. + * + * `num_values` is the number of logical rows to consume from the definition-level stream. + * `selected_indices` uses the same sorted, batch-relative contract as + * ColumnSelectVector::init_from_selection(). Selected nulls append defaults to `doris_column` + * and set bits in `null_map`; unselected non-nulls still advance the encoded payload cursor. + * The method therefore advances exactly `num_values` logical rows while materializing exactly + * `selected_count` rows. + * + * This entry point is deliberately limited to non-repeated leaves. LIST/MAP/STRUCT first build + * a shared level plan because their selection is expressed in parent rows rather than leaf + * level positions. Keeping those contracts separate prevents a flat fast path from silently + * misaligning repeated children. + */ + Status decode_flat_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, + size_t num_values, const uint16_t* selected_indices, + size_t selected_count, NullMap* null_map, bool is_dict_filter); + // Get the repetition level decoder of current page. LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } // Get the definition level decoder of current page. @@ -240,7 +291,7 @@ class ColumnChunkReader { } ColumnChunkReaderState _state = NOT_INIT; - FieldSchema* _field_schema = nullptr; + const ParquetColumnChunkSchema _chunk_schema; const level_t _max_rep_level; const level_t _max_def_level; @@ -278,6 +329,8 @@ class ColumnChunkReader { // Map: encoding -> Decoder // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding std::unordered_map> _decoders; + // Alternating non-null/null runs for decode_flat_values(). Capacity is retained across pages. + std::vector _definition_runs; ColumnChunkReaderStatistics _chunk_statistics; }; diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 377a26ce530db8..74d60aa9c62519 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -108,6 +108,90 @@ instructions as well; this file adds format-v2-specific review expectations. - For JNI readers, review local/global reference lifetime, exception propagation, type conversion, thread attachment assumptions, and cleanup on partial initialization. +### Parquet Native Decode Kernel + +- Keep the production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and + performance baseline, not the migration target. Do not change the v1 reader call path merely to + exercise v2 code. A shared-kernel change under `be/src/format/parquet/` is acceptable only when + extraction cannot reasonably avoid it, the legacy API and behavior remain unchanged, and focused + v1 compatibility tests accompany the v2 tests. +- Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema + objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, + and Dremel-level thresholds. Review constructor arguments and stored references for ownership and + lifetime; metadata owned by a temporary schema adapter must not escape into a persistent reader. +- Treat selection positions as logical Row Group rows, including null rows. Selection indices must + be sorted, unique, and bounded by the batch's logical row count. Dense identity, empty selection, + and fragmented selection must have explicit representations and tests; do not silently mix row + ordinals with non-null value ordinals or dictionary IDs. +- Verify the three decode counts separately: logical rows consumed, encoded non-null payload values + consumed, and output values materialized. Null and filtered-null runs consume no payload; + selected and filtered non-null runs both consume payload. Every page transition, skip, error, and + end-of-batch path must leave all three cursors aligned for the next call. +- A flat scalar fast path may run only when `max_repetition_level == 0`. Repeated leaves require a + shared definition/repetition-level plan that identifies parent-row boundaries, empty and null + collections, null ancestors, and rows spanning pages. Sibling readers in a STRUCT, ARRAY, or MAP + must consume the same parent-row plan rather than independently inferring offsets or null maps. +- Treat the current `load_nested_batch()` / `load_nested_levels_batch()` / + `build_nested_column()` / `consume_nested_column()` split as an Arrow-migration facade, not the + native kernel API. Review new native code for one explicit nested-read request carrying parent + row count, selection, and `VALUES` versus `LEVELS_ONLY` mode. Shape parsing, payload-cursor + advancement, validation, and output must have an obvious owner; callers must not depend on an + undocumented load-before-build phase. +- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one level + traversal and share the result. For example, `[[1, 2], NULL, []]` must yield entry counts + `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning the same levels per child. MAP key + levels own entry existence; value levels validate against that plan. STRUCT siblings validate + alignment but do not independently reconstruct the parent shape. +- Do not size level or selection scratch from a 16-bit batch-row assumption. A repeated parent row + can contain more level entries than the requested parent-row batch. Split large runs without + changing alternation or row-boundary semantics, and check overflow before narrowing counts. +- Decoder dispatch must reject an incompatible physical type, encoding, or type length explicitly. + Review PLAIN, dictionary/RLE, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, BYTE_STREAM_SPLIT, BOOLEAN RLE, and level RLE/bit-packed paths for identical + selection and malformed-input behavior. Page V1 and V2 must feed the same decoder contract after + their different level/decompression layouts are parsed. +- Keep every index coordinate domain explicit: table-local column ID, physical leaf-column ID, + Row Group ID, data-page ordinal, OffsetIndex row ordinal, logical batch-row ordinal, non-null + payload ordinal, and dictionary-entry ID are different types of identity. Data-page ordinals must + exclude dictionary pages consistently. Dictionary-entry bitmaps are local to one Column Chunk + dictionary and cannot be reused after a Row Group, dictionary, or encoding transition. +- Review index composition, not only each index in isolation. Row Group statistics, dictionary, + Bloom, ColumnIndex/OffsetIndex, page cache registration, page skip plans, SelectionVector, and + lazy column cursors must describe the same surviving logical rows. Missing or unusable optional + indexes retain candidates; structurally inconsistent indexes or out-of-range IDs return an + explicit corruption error. A mixed dictionary/plain Column Chunk must leave dictionary-ID + filtering before any cursor is consumed. +- Materialize directly into Doris columns when the physical and target layouts allow it. Decimal + and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, precision, + and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. A direct + path may not bypass the conversion rules used by the fallback path. +- Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across + batches. String-like decoders should gather selected `StringRef` values and append once per batch, + rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded + high-water mark but must be reset logically between pages, Row Groups, files, and errors. +- Adaptive batch sizing must measure completed Doris output rows/bytes and must not recreate native + readers, builders, or scratch when only the row cap changes. Compare the v1 and v2 lifecycle: + probe batches must not turn persistent setup into a per-batch cost or amplify highly fragmented + selection work. +- Footer and metadata caching must key on stable file identity and cache the serialized footer plus + parsed native metadata at the same lifecycle as v1. Never reuse metadata when path, file size, + modification/version identity, encryption state, or schema-affecting options differ. Cache misses + and uncacheable identities must remain correct without a fallback to stale entries. +- Page-cache behavior must match v1 for cache key, stable file identity, registered byte range, + compressed/decompressed entry kind, checksum/decompression ownership, subrange coverage, + invalidation, admission, and fallback I/O. Any intentional difference needs benchmark and memory + evidence showing it is no worse for v1 workloads. Cache lookup must never alter page ordinal, + decoder, level, or dictionary cursor state. +- Preserve observability inside aggregate counters. `TotalBatches` must be decomposable into probe, + dense, selected, empty, page-crossing, and nested/fragmented work where relevant; decode, level, + selection, conversion, allocation, and materialization time must remain attributable without + adding per-row timer overhead. +- Profile index attempts, successes, conservative fallbacks, and corrupt rejections separately for + statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skipping. Footer/page/file/ + condition-cache counters must expose requests, hits, misses, writes/admissions, bytes, wait/I/O + time, and bypass reasons with semantics aligned to v1. A lower total timer without its internal + work counters is not sufficient observability. + ## Detailed FileReader Review Guides - Before reviewing any FileReader implementation, index, predicate path, cache, or virtual column, @@ -174,6 +258,15 @@ instructions as well; this file adds format-v2-specific review expectations. malformed input. - For bug fixes, require a test that fails for the original reachable path and validates the result, row count, or explicit error after the fix. +- For native Parquet work, require a matrix across physical/logical types, every supported encoding, + Page V1/V2, required/optional/repeated levels, dictionary fallback, page and batch boundaries, + dense/empty/fragmented selections, null placement, malformed/truncated input, and files from + representative external writers. Include focused cursor-invariant tests and end-to-end nested + reconstruction tests; a scalar happy-path benchmark is not sufficient coverage. +- Performance-sensitive decoder changes need a reproducible comparison against v1 and a relevant + external baseline such as DuckDB. Report data shape, encoding, selectivity, null/cardinality + distribution, compression, storage path, batch policy, warm/cold cache state, CPU time, rows/s, + bytes/s, allocation behavior, and Profile counter deltas. ## Review Output diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp index c042fc99b512aa..5fc08f8fae31cf 100644 --- a/be/src/format_v2/parquet/reader/list_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/list_column_reader.cpp @@ -100,8 +100,8 @@ Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bo const auto& def_levels = _element_reader->nested_definition_levels(); const auto& rep_levels = _element_reader->nested_repetition_levels(); const int64_t levels_written = _element_reader->nested_levels_written(); - std::vector entry_counts; - NullMap parent_nulls; + _entry_counts.clear(); + _parent_nulls.clear(); *values_processed = 0; int64_t level_idx = nested_build_level_cursor(); const int16_t min_parent_definition_level = @@ -119,12 +119,12 @@ Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bo continue; } if (rep_level == _repetition_level) { - if (entry_counts.empty()) { + if (_entry_counts.empty()) { return Status::Corruption("Invalid repeated level for parquet LIST column {}", _name); } if (def_level >= _definition_level) { - ++entry_counts.back(); + ++_entry_counts.back(); } continue; } @@ -134,8 +134,8 @@ Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bo return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", _name); } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); + _parent_nulls.push_back(parent_is_null); + _entry_counts.push_back(def_level >= _definition_level ? 1 : 0); ++*values_processed; } set_nested_build_level_cursor(level_idx); @@ -143,7 +143,7 @@ Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bo uint64_t total_entries = 0; int64_t child_value_count = 0; if (!_element_reader->is_or_has_repeated_child()) { - for (const auto entry_count : entry_counts) { + for (const auto entry_count : _entry_counts) { total_entries += entry_count; } if (column != nullptr) { @@ -178,7 +178,7 @@ Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bo return Status::OK(); }; - for (const auto entry_count : entry_counts) { + for (const auto entry_count : _entry_counts) { total_entries += entry_count; if (entry_count > 0) { pending_entries += entry_count; @@ -195,8 +195,8 @@ Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bo } if (column != nullptr) { array_column->get_data_ptr() = std::move(nested_column); - append_offsets(array_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); + append_offsets(array_column->get_offsets(), _entry_counts); + append_parent_nulls(parent_null_map, _parent_nulls); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h index d64be546e394d7..ac3d44651846d8 100644 --- a/be/src/format_v2/parquet/reader/list_column_reader.h +++ b/be/src/format_v2/parquet/reader/list_column_reader.h @@ -52,6 +52,10 @@ class ListColumnReader final : public ParquetColumnReader { std::unique_ptr _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) + // A list row such as [[1, 2], NULL, []] becomes entry counts [2, 0, 0] and parent nulls + // [0, 1, 0]. These vectors are batch scratch, not reader state; clear() keeps their capacity. + std::vector _entry_counts; + NullMap _parent_nulls; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp index 8217d0c013abc0..0d165b01e2fb52 100644 --- a/be/src/format_v2/parquet/reader/map_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/map_column_reader.cpp @@ -110,9 +110,9 @@ Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bou const auto& rep_levels = _key_reader->nested_repetition_levels(); const int64_t levels_written = _key_reader->nested_levels_written(); - std::vector entry_counts; - std::vector map_level_indices; - NullMap parent_nulls; + _entry_counts.clear(); + _map_level_indices.clear(); + _parent_nulls.clear(); *values_processed = 0; int64_t level_idx = nested_build_level_cursor(); const int16_t min_parent_definition_level = @@ -130,14 +130,14 @@ Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bou (!starts_parent && def_level < _repeated_ancestor_definition_level)) { continue; } - map_level_indices.push_back(current_level_idx); + _map_level_indices.push_back(current_level_idx); if (rep_level == _repetition_level) { - if (entry_counts.empty()) { + if (_entry_counts.empty()) { return Status::Corruption("Invalid repeated level for parquet MAP column {}", _name); } if (def_level >= _definition_level) { - ++entry_counts.back(); + ++_entry_counts.back(); } continue; } @@ -147,14 +147,14 @@ Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bou return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", _name); } - parent_nulls.push_back(parent_is_null); - entry_counts.push_back(def_level >= _definition_level ? 1 : 0); + _parent_nulls.push_back(parent_is_null); + _entry_counts.push_back(def_level >= _definition_level ? 1 : 0); ++*values_processed; } set_nested_build_level_cursor(level_idx); uint64_t total_entries = 0; - for (const auto entry_count : entry_counts) { + for (const auto entry_count : _entry_counts) { total_entries += entry_count; } int64_t key_value_count = 0; @@ -166,7 +166,7 @@ Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bou } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { // MAP keys are required even if a projected Doris key type is nullable. Validate each // actual entry directly from the key level stream while advancing past empty/null maps. - for (const int64_t key_level_idx : map_level_indices) { + for (const int64_t key_level_idx : _map_level_indices) { if (def_levels[key_level_idx] >= _definition_level) { RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); ++key_value_count; @@ -194,7 +194,7 @@ Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bou const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); - for (const int64_t key_level_idx : map_level_indices) { + for (const int64_t key_level_idx : _map_level_indices) { while (value_level_idx < value_levels_written && (value_rep_levels[value_level_idx] > _repetition_level || value_def_levels[value_level_idx] < min_parent_definition_level || @@ -248,8 +248,8 @@ Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bou if (column != nullptr) { map_column->get_keys_ptr() = std::move(key_column); map_column->get_values_ptr() = std::move(value_column); - append_offsets(map_column->get_offsets(), entry_counts); - append_parent_nulls(parent_null_map, parent_nulls); + append_offsets(map_column->get_offsets(), _entry_counts); + append_parent_nulls(parent_null_map, _parent_nulls); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h index 1a8ca9c70d8c5b..df81b29137f152 100644 --- a/be/src/format_v2/parquet/reader/map_column_reader.h +++ b/be/src/format_v2/parquet/reader/map_column_reader.h @@ -56,6 +56,11 @@ class MapColumnReader final : public ParquetColumnReader { std::unique_ptr _key_reader; // key column reader (always read fully) std::unique_ptr _value_reader; // value column reader (can be pruned by projection) + // Key levels own MAP entry existence. The level index vector lets the value stream advance in + // lockstep without building another per-row lookup table; all three buffers retain capacity. + std::vector _entry_counts; + std::vector _map_level_indices; + NullMap _parent_nulls; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp index fd261ef5219d27..e8aec17605eb92 100644 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp @@ -163,9 +163,13 @@ Status append_dictionary_binary_values(const std::string& column_name, Status build_binary_values(const std::string& column_name, const std::vector>& chunks, int64_t records_read, const NullMap* null_map, - bool read_dense_for_nullable, std::vector* binary_values) { - std::vector compact_values; - auto* values = read_dense_for_nullable ? &compact_values : binary_values; + bool read_dense_for_nullable, std::vector* binary_values, + std::vector* compact_values) { + DORIS_CHECK(binary_values != nullptr); + DORIS_CHECK(compact_values != nullptr); + binary_values->clear(); + compact_values->clear(); + auto* values = read_dense_for_nullable ? compact_values : binary_values; values->reserve(records_read); for (const auto& chunk : chunks) { if (chunk == nullptr) { @@ -207,11 +211,11 @@ Status build_binary_values(const std::string& column_name, } const int64_t non_null_count = static_cast(simd::count_zero_num( reinterpret_cast(null_map->data()), null_map->size())); - if (compact_values.size() != static_cast(non_null_count)) { + if (compact_values->size() != static_cast(non_null_count)) { return Status::Corruption( "Invalid dense nullable parquet binary values for column {}: values={}, " "records={}, nulls={}", - column_name, compact_values.size(), records_read, + column_name, compact_values->size(), records_read, records_read - non_null_count); } binary_values->reserve(records_read); @@ -221,7 +225,7 @@ Status build_binary_values(const std::string& column_name, binary_values->emplace_back(static_cast(nullptr), 0); continue; } - binary_values->emplace_back(compact_values[value_idx++]); + binary_values->emplace_back((*compact_values)[value_idx++]); } return Status::OK(); } @@ -349,8 +353,8 @@ Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value // materialization. if (batch->is_binary_value()) { - std::vector> discarded_chunks; - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &discarded_chunks)); + _discarded_binary_chunks.clear(); + RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &_discarded_binary_chunks)); } // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the @@ -363,9 +367,20 @@ Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader // - FLOAT16: binary -> half_to_float -> float_values Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, MutableColumnPtr& column) const { - std::vector binary_values; - std::vector spaced_values; - std::vector float_values; + return append_values_with_type(batch, row_count, null_map, _type, _serde, column); +} + +Status ParquetLeafReader::append_values_with_type(const ParquetLeafBatch& batch, int64_t row_count, + const NullMap* null_map, + const DataTypePtr& materialization_type, + const DataTypeSerDeSPtr& serde, + MutableColumnPtr& column) const { + DORIS_CHECK(materialization_type != nullptr); + DORIS_CHECK(serde != nullptr); + _binary_values.clear(); + _compact_binary_values.clear(); + _spaced_values.clear(); + _float_values.clear(); DecodedColumnView view; view.value_kind = batch._value_kind; view.time_unit = decoded_time_unit(_type_descriptor.time_unit); @@ -383,18 +398,20 @@ Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t r if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, binary_values, row_count, - &float_values)); + read_dense_for_nullable, &_binary_values, + &_compact_binary_values)); + RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, _binary_values, row_count, + &_float_values)); view.value_kind = DecodedValueKind::FLOAT; - view.values = reinterpret_cast(float_values.data()); + view.values = reinterpret_cast(_float_values.data()); } else if (batch.is_binary_value()) { RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &binary_values)); - view.binary_values = &binary_values; + read_dense_for_nullable, &_binary_values, + &_compact_binary_values)); + view.binary_values = &_binary_values; } else if (read_dense_for_nullable) { - RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &spaced_values)); - view.values = spaced_values.data(); + RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &_spaced_values)); + view.values = _spaced_values.data(); } else { view.values = batch._fixed_values; } @@ -405,14 +422,14 @@ Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t r { SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { + if (!materialization_type->is_nullable()) { if (auto* nullable_column = check_and_get_column(*column); nullable_column != nullptr) { auto& nested_column = nullable_column->get_nested_column(); auto& tmp_null_map = nullable_column->get_null_map_data(); const auto old_nested_size = nested_column.size(); const auto old_null_map_size = tmp_null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); + auto st = serde->read_column_from_decoded_values(nested_column, view); if (!st.ok()) { nested_column.resize(old_nested_size); return st; @@ -421,10 +438,10 @@ Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t r memset(tmp_null_map.data() + old_null_map_size, 0, tmp_null_map.size() - old_null_map_size); } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); + RETURN_IF_ERROR(serde->read_column_from_decoded_values(*column, view)); } } else { - RETURN_IF_ERROR(_type->get_serde()->read_column_from_decoded_values(*column, view)); + RETURN_IF_ERROR(serde->read_column_from_decoded_values(*column, view)); } } return Status::OK(); @@ -478,7 +495,10 @@ ParquetLeafReader::ParquetLeafReader( _profile(profile), _timezone(timezone), _enable_strict_mode(enable_strict_mode), - _decoded_value_appender(std::move(decoded_value_appender)) {} + _decoded_value_appender(std::move(decoded_value_appender)), + _serde(_type->get_serde()), + _nested_value_type(remove_nullable(_type)), + _nested_value_serde(_nested_value_type->get_serde()) {} Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const { @@ -536,11 +556,11 @@ Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, int16_t value_slot_repetition_level) const { - ParquetLeafBatch leaf_batch; int64_t records_read = 0; - RETURN_IF_ERROR(read_batch(batch_rows, &leaf_batch, &records_read)); - return build_nested_batch_from_leaf_batch(leaf_batch, records_read, value_slot_definition_level, - batch, value_slot_repetition_level); + RETURN_IF_ERROR(read_batch(batch_rows, &_nested_leaf_batch, &records_read)); + return build_nested_batch_from_leaf_batch(_nested_leaf_batch, records_read, + value_slot_definition_level, batch, + value_slot_repetition_level); } Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, @@ -554,7 +574,6 @@ Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, } int64_t records_read = 0; - ParquetLeafBatch leaf_batch; try { _record_reader->Reset(); _record_reader->Reserve(batch_rows); @@ -573,8 +592,8 @@ Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, records_read); } - RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &leaf_batch)); - return build_nested_levels_batch_from_leaf_batch(leaf_batch, records_read, batch); + RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &_nested_leaf_batch)); + return build_nested_levels_batch_from_leaf_batch(_nested_leaf_batch, records_read, batch); } Status ParquetLeafReader::build_nested_batch_from_leaf_batch( @@ -584,7 +603,7 @@ Status ParquetLeafReader::build_nested_batch_from_leaf_batch( if (batch == nullptr) { return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); } - *batch = ParquetNestedScalarBatch(); + batch->reset(); batch->value_slot_definition_level = value_slot_definition_level; batch->value_slot_repetition_level = value_slot_repetition_level; @@ -694,7 +713,8 @@ Status ParquetLeafReader::build_nested_batch_from_leaf_batch( } batch->value_indices.resize(static_cast(batch->levels_written), -1); - NullMap value_nulls(static_cast(values_written), 1); + _nested_value_nulls.resize(static_cast(values_written)); + std::fill(_nested_value_nulls.begin(), _nested_value_nulls.end(), 1); int64_t value_idx = 0; const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS ? payload_slot_definition_level @@ -722,7 +742,7 @@ Status ParquetLeafReader::build_nested_batch_from_leaf_batch( DORIS_CHECK(decoded_value_idx < values_written); if (has_leaf_value) { batch->value_indices[static_cast(level_idx)] = decoded_value_idx; - value_nulls[static_cast(decoded_value_idx)] = 0; + _nested_value_nulls[static_cast(decoded_value_idx)] = 0; } } if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { @@ -732,13 +752,13 @@ Status ParquetLeafReader::build_nested_batch_from_leaf_batch( _name, values_written, value_idx); } - const auto value_type = remove_nullable(_type); - batch->values_column = value_type->create_column(); + if (batch->values_column.get() == nullptr) { + batch->values_column = _nested_value_type->create_column(); + } if (values_written > 0) { - ParquetLeafReader value_reader(_descriptor, _type_descriptor, value_type, _name, - _record_reader, _profile, _timezone, _enable_strict_mode); - RETURN_IF_ERROR(value_reader.append_values(leaf_batch, values_written, &value_nulls, - batch->values_column)); + RETURN_IF_ERROR(append_values_with_type(leaf_batch, values_written, &_nested_value_nulls, + _nested_value_type, _nested_value_serde, + batch->values_column)); } return Status::OK(); } @@ -749,7 +769,7 @@ Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( if (batch == nullptr) { return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); } - *batch = ParquetNestedScalarBatch(); + batch->reset(); batch->records_read = records_read; batch->levels_written = leaf_batch.consumed_level_count(); if (batch->levels_written > leaf_batch.decoded_level_count()) { diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h index b396b35fd1f32c..2b2f6121027201 100644 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h @@ -25,7 +25,9 @@ #include "common/status.h" #include "core/column/column.h" #include "core/column/column_nullable.h" +#include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/string_ref.h" #include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/parquet_type.h" @@ -63,6 +65,23 @@ struct ParquetNestedScalarBatch { MutableColumnPtr values_column; bool empty() const { return levels_written == 0; } + + // Reset logical contents without replacing the vectors/Column object. A reader repeatedly + // sees the same physical leaf type, so retaining capacity is safe and avoids one allocation + // family per nested batch. A levels-only read leaves values_column empty but may keep the + // reusable object allocated. + void reset() { + records_read = 0; + levels_written = 0; + value_slot_definition_level = 0; + value_slot_repetition_level = std::numeric_limits::max(); + def_levels.clear(); + rep_levels.clear(); + value_indices.clear(); + if (values_column.get() != nullptr) { + values_column->clear(); + } + } }; class ParquetLeafBatch { @@ -147,6 +166,10 @@ class ParquetLeafReader { const NullMap* null_map, std::vector* spaced_values) const; + Status append_values_with_type(const ParquetLeafBatch& batch, int64_t row_count, + const NullMap* null_map, const DataTypePtr& materialization_type, + const DataTypeSerDeSPtr& serde, MutableColumnPtr& column) const; + Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, int64_t records_read, int16_t value_slot_definition_level, @@ -168,6 +191,18 @@ class ParquetLeafReader { const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion bool _enable_strict_mode = false; // strict mode for type mismatch errors std::function _decoded_value_appender; + // Logical scratch. append_values() resets sizes but preserves capacity across batches. + // StringRef entries never outlive the Arrow chunks retained by the current leaf batch. + DataTypeSerDeSPtr _serde; + DataTypePtr _nested_value_type; + DataTypeSerDeSPtr _nested_value_serde; + mutable ParquetLeafBatch _nested_leaf_batch; + mutable NullMap _nested_value_nulls; + mutable std::vector _binary_values; + mutable std::vector _compact_binary_values; + mutable std::vector _spaced_values; + mutable std::vector _float_values; + mutable std::vector> _discarded_binary_chunks; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp index 520a3cb1e62e3b..d34a8b94b29842 100644 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp @@ -126,6 +126,9 @@ ScalarColumnReader::ScalarColumnReader( _descriptor(column_schema.descriptor), _type_descriptor(column_schema.type_descriptor), _record_reader(std::move(record_reader)), + _leaf_reader(std::make_unique(_descriptor, _type_descriptor, _type, + _name, _record_reader, _profile, + timezone, enable_strict_mode)), _page_skip_plan(page_skip_plan), _timezone(timezone), _enable_strict_mode(enable_strict_mode), @@ -142,33 +145,32 @@ Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* return Status::InternalError("Parquet record reader is not initialized for column {}", _name); } - auto reader = leaf_reader(); - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(reader.read_batch(rows, &leaf_batch, rows_read)); + auto& reader = leaf_reader(); + RETURN_IF_ERROR(reader.read_batch(rows, &_leaf_batch, rows_read)); - NullMap null_map; - RETURN_IF_ERROR(reader.build_null_map(leaf_batch, *rows_read, &null_map)); + _null_map.clear(); + RETURN_IF_ERROR(reader.build_null_map(_leaf_batch, *rows_read, &_null_map)); const auto value_kind = decoded_value_kind(_type_descriptor); const bool is_binary_value = value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; - if (!is_binary_value && leaf_batch.read_dense_for_nullable() && !null_map.empty()) { + if (!is_binary_value && _leaf_batch.read_dense_for_nullable() && !_null_map.empty()) { const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map.data()), null_map.size())); + reinterpret_cast(_null_map.data()), _null_map.size())); const int64_t null_count = *rows_read - non_null_count; - if (leaf_batch.values_written() != non_null_count) { + if (_leaf_batch.values_written() != non_null_count) { return Status::Corruption( "Invalid dense nullable parquet record read result for column {}: values={}, " "records={}, nulls={}", - _name, leaf_batch.values_written(), *rows_read, null_count); + _name, _leaf_batch.values_written(), *rows_read, null_count); } - } else if (!is_binary_value && !leaf_batch.read_dense_for_nullable() && - leaf_batch.values_written() != *rows_read) { + } else if (!is_binary_value && !_leaf_batch.read_dense_for_nullable() && + _leaf_batch.values_written() != *rows_read) { return Status::Corruption( "Invalid parquet record read result for column {}: values={}, records={}", _name, - leaf_batch.values_written(), *rows_read); + _leaf_batch.values_written(), *rows_read); } - RETURN_IF_ERROR(reader.append_values(leaf_batch, *rows_read, &null_map, column)); + RETURN_IF_ERROR(reader.append_values(_leaf_batch, *rows_read, &_null_map, column)); advance_rows_read(*rows_read); update_reader_read_rows(*rows_read); return Status::OK(); @@ -274,9 +276,9 @@ Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& } *used_filter = true; - const auto ranges = selection_to_ranges(sel, selected_rows); + selection_to_ranges(sel, selected_rows, &_selection_ranges); int64_t cursor = 0; - for (const auto& range : ranges) { + for (const auto& range : _selection_ranges) { if (range.start < cursor || range.start + range.length > batch_rows) { return Status::InvalidArgument( "Invalid parquet dictionary selection range [{}, {}) for column {}", @@ -320,12 +322,11 @@ Status ScalarColumnReader::read_range_with_dictionary_filter( _name); } - ParquetLeafBatch leaf_batch; - RETURN_IF_ERROR(leaf_reader().read_batch(rows, &leaf_batch, rows_read)); + RETURN_IF_ERROR(leaf_reader().read_batch(rows, &_leaf_batch, rows_read)); int64_t matched_rows = 0; - RETURN_IF_ERROR(append_dictionary_filtered_values(leaf_batch.binary_chunks(), dictionary_filter, - column, row_filter, &matched_rows, - used_filter)); + RETURN_IF_ERROR(append_dictionary_filtered_values(_leaf_batch.binary_chunks(), + dictionary_filter, column, row_filter, + &matched_rows, used_filter)); if (!*used_filter) { return Status::Corruption( "Parquet dictionary reader did not return dictionary batches for column {}", _name); @@ -344,14 +345,14 @@ Status ScalarColumnReader::read_range_with_dictionary_filter( Status ScalarColumnReader::append_dictionary_filtered_values( const std::vector>& chunks, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const { + IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { DORIS_CHECK(row_filter != nullptr); DORIS_CHECK(matched_rows != nullptr); DORIS_CHECK(used_filter != nullptr); *matched_rows = 0; *used_filter = false; - std::vector selected_values; + _dictionary_binary_values.clear(); for (const auto& chunk : chunks) { DORIS_CHECK(chunk != nullptr); const auto* dict_array = dynamic_cast(chunk.get()); @@ -386,7 +387,7 @@ Status ScalarColumnReader::append_dictionary_filtered_values( keep = dictionary_filter[static_cast(dictionary_index)] != 0; if (keep) { RETURN_IF_ERROR(append_arrow_binary_dictionary_value( - _name, *dictionary, dictionary_index, &selected_values)); + _name, *dictionary, dictionary_index, &_dictionary_binary_values)); ++*matched_rows; } } @@ -397,7 +398,11 @@ Status ScalarColumnReader::append_dictionary_filtered_values( if (!*used_filter) { return Status::OK(); } - return append_decoded_binary_values(selected_values, column); + auto status = append_decoded_binary_values(_dictionary_binary_values, column); + // StringRef does not own the Arrow dictionary bytes. Drop the logical references immediately + // after synchronous materialization while keeping vector capacity for the next batch. + _dictionary_binary_values.clear(); + return status; } Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h index 5342baa803eca0..ae611402aa0648 100644 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.h +++ b/be/src/format_v2/parquet/reader/scalar_column_reader.h @@ -40,7 +40,9 @@ namespace doris::format::parquet { struct ScalarColumnReaderTestAccess; -// load_nested_batch() / build_nested_column() +// Owns the physical leaf reader for the lifetime of the row-group reader. Keeping this object +// stable is important: ParquetLeafReader owns reusable SerDe and conversion scratch whose +// capacity would otherwise be discarded after every batch. class ScalarColumnReader final : public ParquetColumnReader { friend class MapColumnReader; friend struct ScalarColumnReaderTestAccess; @@ -80,15 +82,15 @@ class ScalarColumnReader final : public ParquetColumnReader { Status append_dictionary_filtered_values( const std::vector>& chunks, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) const; + IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter); Status append_decoded_binary_values(const std::vector& values, MutableColumnPtr& column) const; const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } - ParquetLeafReader leaf_reader() const { - return ParquetLeafReader(_descriptor, _type_descriptor, _type, _name, _record_reader, - _profile, _timezone, _enable_strict_mode); + ParquetLeafReader& leaf_reader() { + DORIS_CHECK(_leaf_reader != nullptr); + return *_leaf_reader; } void advance_rows_read(int64_t rows); @@ -98,12 +100,18 @@ class ScalarColumnReader final : public ParquetColumnReader { const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor ParquetTypeDescriptor _type_descriptor; // type encoding information std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader + _record_reader; // Arrow physical column reader + std::unique_ptr _leaf_reader; // persistent leaf decoder/materializer const ParquetPageSkipPlan* _page_skip_plan = nullptr; // page-index pruning result (may be nullptr) const cctz::time_zone* _timezone = nullptr; // timezone bool _enable_strict_mode = false; // strict mode int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) + // Per-batch logical scratch. Each operation clears the logical size while retaining capacity. + ParquetLeafBatch _leaf_batch; + NullMap _null_map; + std::vector _selection_ranges; + std::vector _dictionary_binary_values; std::unique_ptr _nested_batch; // intermediate result for nested reads }; diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp index 5abe7abe75e9a2..3df6437501934d 100644 --- a/be/src/format_v2/parquet/reader/struct_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/struct_column_reader.cpp @@ -152,8 +152,8 @@ Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_ const auto& rep_levels = shape_reader->nested_repetition_levels(); const int64_t levels_written = shape_reader->nested_levels_written(); - NullMap parent_nulls; - std::vector parent_level_indices; + _parent_nulls.clear(); + _parent_level_indices.clear(); *values_processed = 0; int64_t level_idx = nested_build_level_cursor(); while (level_idx < levels_written) { @@ -177,17 +177,17 @@ Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_ return Status::Corruption( "Parquet STRUCT column {} contains null for non-nullable struct", _name); } - parent_nulls.push_back(parent_is_null); - parent_level_indices.push_back(current_level_idx); + _parent_nulls.push_back(parent_is_null); + _parent_level_indices.push_back(current_level_idx); ++*values_processed; } set_nested_build_level_cursor(level_idx); - std::vector child_columns; + _child_columns.clear(); if (column != nullptr) { - child_columns.reserve(struct_column->get_columns().size()); + _child_columns.reserve(struct_column->get_columns().size()); for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { - child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); + _child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); } } for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { @@ -208,7 +208,7 @@ Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_ int64_t child_rows = 0; if (column != nullptr) { RETURN_IF_ERROR(_children[child_idx]->build_nested_column( - pending_present_rows, child_columns[output_idx], &child_rows)); + pending_present_rows, _child_columns[output_idx], &child_rows)); } else { RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, &child_rows)); @@ -222,18 +222,18 @@ Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_ pending_present_rows = 0; return Status::OK(); }; - for (size_t parent_idx = 0; parent_idx < parent_nulls.size(); ++parent_idx) { - const auto parent_is_null = parent_nulls[parent_idx]; + for (size_t parent_idx = 0; parent_idx < _parent_nulls.size(); ++parent_idx) { + const auto parent_is_null = _parent_nulls[parent_idx]; if (!parent_is_null) { ++pending_present_rows; continue; } RETURN_IF_ERROR(flush_present_rows()); if (column != nullptr) { - child_columns[output_idx]->insert_default(); + _child_columns[output_idx]->insert_default(); } RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), - parent_level_indices[parent_idx])); + _parent_level_indices[parent_idx])); ++total_child_rows; } RETURN_IF_ERROR(flush_present_rows()); @@ -244,10 +244,10 @@ Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_ } } if (column != nullptr) { - for (size_t child_idx = 0; child_idx < child_columns.size(); ++child_idx) { - struct_column->get_column_ptr(child_idx) = std::move(child_columns[child_idx]); + for (size_t child_idx = 0; child_idx < _child_columns.size(); ++child_idx) { + struct_column->get_column_ptr(child_idx) = std::move(_child_columns[child_idx]); } - append_parent_nulls(parent_null_map, parent_nulls); + append_parent_nulls(parent_null_map, _parent_nulls); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h index 3c2d6904cb36f4..57e66e1e2236fd 100644 --- a/be/src/format_v2/parquet/reader/struct_column_reader.h +++ b/be/src/format_v2/parquet/reader/struct_column_reader.h @@ -60,6 +60,11 @@ class StructColumnReader final : public ParquetColumnReader { std::vector> _children; // projected child readers std::vector _child_output_indices; // child reader -> struct output position mapping + // Parent shape is derived once from the chosen shape child, then reused for every projected + // child. These buffers preserve capacity across batches and avoid allocating per child. + NullMap _parent_nulls; + std::vector _parent_level_indices; + std::vector _child_columns; }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 589154d4acc0e4..8d3a3dd4accfa4 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -137,11 +137,12 @@ class SelectionVector { size_t _size = 0; }; -inline std::vector selection_to_ranges(const SelectionVector& selection, - uint16_t selected_rows) { - std::vector ranges; +inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, + std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); if (selected_rows == 0) { - return ranges; + return; } int64_t range_start = selection.get_index(0); @@ -152,11 +153,17 @@ inline std::vector selection_to_ranges(const SelectionVector& selectio previous = current; continue; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); range_start = current; previous = current; } - ranges.push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); + ranges->push_back(RowRange {.start = range_start, .length = previous - range_start + 1}); +} + +inline std::vector selection_to_ranges(const SelectionVector& selection, + uint16_t selected_rows) { + std::vector ranges; + selection_to_ranges(selection, selected_rows, &ranges); return ranges; } diff --git a/be/test/format/parquet/byte_array_dict_decoder_test.cpp b/be/test/format/parquet/byte_array_dict_decoder_test.cpp index 2e3f398ef56e5a..5c2f2b75b91657 100644 --- a/be/test/format/parquet/byte_array_dict_decoder_test.cpp +++ b/be/test/format/parquet/byte_array_dict_decoder_test.cpp @@ -178,6 +178,33 @@ TEST_F(ByteArrayDictDecoderTest, test_decode_with_filter_and_null) { } } +TEST_F(ByteArrayDictDecoderTest, test_fragmented_index_selection_with_nulls) { + MutableColumnPtr column = ColumnString::create(); + DataTypePtr data_type = std::make_shared(); + + // Four dictionary zeroes followed by dictionary index 2. Null logical rows consume no index. + std::vector rle_data = {2, 8, 0, 3, 0b00000010, 0}; + Slice data_slice(reinterpret_cast(rle_data.data()), rle_data.size()); + ASSERT_TRUE(_decoder.set_data(&data_slice).ok()); + + const std::vector null_runs = {4, 1, 1, 1}; + const std::vector selection = {1, 4, 5, 6}; + NullMap null_map; + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 7, &null_map, selection.data(), + selection.size()) + .ok()); + + ASSERT_TRUE(_decoder.decode_values(column, data_type, select_vector, false).ok()); + ASSERT_EQ(column->size(), 4); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1})); + EXPECT_EQ(column->get_data_at(0).to_string(), "apple"); + EXPECT_EQ(column->get_data_at(1).to_string(), ""); + EXPECT_EQ(column->get_data_at(2).to_string(), "cherry"); + EXPECT_EQ(column->get_data_at(3).to_string(), ""); +} + // Test empty dictionary case TEST_F(ByteArrayDictDecoderTest, test_empty_dict) { ByteArrayDictDecoder empty_decoder; diff --git a/be/test/format/parquet/byte_array_plain_decoder_test.cpp b/be/test/format/parquet/byte_array_plain_decoder_test.cpp index ad78492a345ccd..cba01b83511578 100644 --- a/be/test/format/parquet/byte_array_plain_decoder_test.cpp +++ b/be/test/format/parquet/byte_array_plain_decoder_test.cpp @@ -244,6 +244,45 @@ TEST_F(ByteArrayPlainDecoderTest, test_skip_value) { EXPECT_EQ(result_column->get_data_at(0).to_string(), "cherry"); } +TEST_F(ByteArrayPlainDecoderTest, test_fragmented_index_selection_with_nulls) { + const char* values[] = {"zero", "two", "three", "five"}; + size_t data_size = 0; + for (const auto* value : values) { + data_size += sizeof(uint32_t) + strlen(value); + } + _data = std::make_unique(data_size); + size_t offset = 0; + for (const auto* value : values) { + const auto length = static_cast(strlen(value)); + encode_fixed32_le(_data.get() + offset, length); + offset += sizeof(uint32_t); + memcpy(_data.get() + offset, value, length); + offset += length; + } + _data_slice = Slice(_data.get(), data_size); + + ByteArrayPlainDecoder decoder; + ASSERT_TRUE(decoder.set_data(&_data_slice).ok()); + MutableColumnPtr column = ColumnString::create(); + DataTypePtr data_type = std::make_shared(); + const std::vector null_runs = {1, 1, 2, 1, 1}; + const std::vector selection = {1, 2, 4, 5}; + NullMap null_map; + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 6, &null_map, selection.data(), + selection.size()) + .ok()); + + ASSERT_TRUE(decoder.decode_values(column, data_type, select_vector, false).ok()); + ASSERT_EQ(column->size(), 4); + EXPECT_EQ(null_map, (NullMap {1, 0, 1, 0})); + EXPECT_EQ(column->get_data_at(0).to_string(), ""); + EXPECT_EQ(column->get_data_at(1).to_string(), "two"); + EXPECT_EQ(column->get_data_at(2).to_string(), ""); + EXPECT_EQ(column->get_data_at(3).to_string(), "five"); +} + TEST_F(ByteArrayPlainDecoderTest, test_decode_truncated_length_prefix) { uint8_t data[] = {0x01, 0x00}; _data_slice = Slice(data, sizeof(data)); diff --git a/be/test/format/parquet/byte_stream_split_decoder_test.cpp b/be/test/format/parquet/byte_stream_split_decoder_test.cpp index b332a13449a99d..d298cf5205dd48 100644 --- a/be/test/format/parquet/byte_stream_split_decoder_test.cpp +++ b/be/test/format/parquet/byte_stream_split_decoder_test.cpp @@ -168,6 +168,38 @@ TEST_F(ByteStreamSplitDecoderTest, test_basic_decode_fixed_length_object) { EXPECT_EQ(fixed_length_value(*result_column, 2), "ghi"); } +TEST_F(ByteStreamSplitDecoderTest, test_fragmented_index_selection_with_nulls) { + const std::vector values = {1.0F, 2.0F, 3.0F, 4.0F}; + std::vector encoded(values.size() * sizeof(float)); + for (size_t value_index = 0; value_index < values.size(); ++value_index) { + const auto* bytes = reinterpret_cast(&values[value_index]); + for (size_t byte_index = 0; byte_index < sizeof(float); ++byte_index) { + encoded[byte_index * values.size() + value_index] = bytes[byte_index]; + } + } + Slice data_slice(encoded.data(), encoded.size()); + ASSERT_TRUE(_decoder.set_data(&data_slice).ok()); + _decoder.set_type_length(sizeof(float)); + + MutableColumnPtr column = ColumnFloat32::create(); + DataTypePtr data_type = std::make_shared(); + const std::vector null_runs = {2, 1, 2, 1}; + const std::vector selection = {1, 2, 4, 5}; + NullMap null_map; + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 6, &null_map, selection.data(), + selection.size()) + .ok()); + + ASSERT_TRUE(_decoder.decode_values(column, data_type, select_vector, false).ok()); + ASSERT_EQ(column->size(), 4); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1})); + const auto& decoded = assert_cast(*column).get_data(); + EXPECT_FLOAT_EQ(decoded[0], 2.0F); + EXPECT_FLOAT_EQ(decoded[2], 4.0F); +} + // Test decoding with filter for FLOAT type TEST_F(ByteStreamSplitDecoderTest, test_decode_with_filter_float) { // Prepare test data for FLOAT type diff --git a/be/test/format/parquet/delta_bit_pack_decoder_test.cpp b/be/test/format/parquet/delta_bit_pack_decoder_test.cpp index 78a857b3442468..8a0a2bd1940e64 100644 --- a/be/test/format/parquet/delta_bit_pack_decoder_test.cpp +++ b/be/test/format/parquet/delta_bit_pack_decoder_test.cpp @@ -157,6 +157,32 @@ TEST_F(DeltaBitPackDecoderTest, test_decode_with_filter_and_null) { } } +TEST_F(DeltaBitPackDecoderTest, test_fragmented_index_selection_with_nulls) { + std::vector encoded_data = { + // Four values 10, 11, 12, 13 with a constant delta of one. + 0x80, 0x01, 0x04, 0x04, 0x14, 0x02, 0x00, 0x00, 0x00, 0x00}; + Slice data_slice(reinterpret_cast(encoded_data.data()), encoded_data.size()); + ASSERT_TRUE(_decoder->set_data(&data_slice).ok()); + + MutableColumnPtr column = ColumnInt32::create(); + DataTypePtr data_type = std::make_shared(); + const std::vector null_runs = {2, 1, 2, 1}; + const std::vector selection = {1, 2, 4, 5}; + NullMap null_map; + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 6, &null_map, selection.data(), + selection.size()) + .ok()); + + ASSERT_TRUE(_decoder->decode_values(column, data_type, select_vector, false).ok()); + ASSERT_EQ(column->size(), 4); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1})); + const auto& decoded = assert_cast(*column).get_data(); + EXPECT_EQ(decoded[0], 11); + EXPECT_EQ(decoded[2], 13); +} + // Test skipping values for delta bit pack decoding TEST_F(DeltaBitPackDecoderTest, test_skip_value) { // Prepare encoded data diff --git a/be/test/format/parquet/fix_length_plain_decoder_test.cpp b/be/test/format/parquet/fix_length_plain_decoder_test.cpp index 78b992c2a36416..e431385a138b9e 100644 --- a/be/test/format/parquet/fix_length_plain_decoder_test.cpp +++ b/be/test/format/parquet/fix_length_plain_decoder_test.cpp @@ -240,6 +240,35 @@ TEST_F(FixLengthPlainDecoderTest, test_decode_with_filter_and_null) { } } +TEST_F(FixLengthPlainDecoderTest, test_fragmented_index_selection_with_nulls) { + int32_t values[] = {123, 456, 789, 1011}; + _data = std::make_unique(sizeof(values)); + memcpy(_data.get(), values, sizeof(values)); + _data_slice = Slice(_data.get(), sizeof(values)); + + FixLengthPlainDecoder decoder; + decoder.set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder.set_data(&_data_slice).ok()); + + MutableColumnPtr column = ColumnInt32::create(); + DataTypePtr data_type = std::make_shared(); + const std::vector null_runs = {1, 1, 2, 1, 1}; + const std::vector selection = {1, 2, 4, 5}; + NullMap null_map; + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 6, &null_map, selection.data(), + selection.size()) + .ok()); + + ASSERT_TRUE(decoder.decode_values(column, data_type, select_vector, false).ok()); + ASSERT_EQ(column->size(), 4); + EXPECT_EQ(null_map, (NullMap {1, 0, 1, 0})); + const auto& decoded = assert_cast(*column).get_data(); + EXPECT_EQ(decoded[1], 456); + EXPECT_EQ(decoded[3], 1011); +} + // Test skipping values TEST_F(FixLengthPlainDecoderTest, test_skip_value) { // Prepare test data: create fixed-length integer values diff --git a/be/test/format/parquet/parquet_common_test.cpp b/be/test/format/parquet/parquet_common_test.cpp index 2bd0dcc253a16c..7ac8f168f5bc54 100644 --- a/be/test/format/parquet/parquet_common_test.cpp +++ b/be/test/format/parquet/parquet_common_test.cpp @@ -454,4 +454,220 @@ TEST_F(ColumnSelectVectorTest, test_filter_map_index) { EXPECT_EQ(type, ColumnSelectVector::CONTENT); } +TEST_F(ColumnSelectVectorTest, test_index_selection_merges_selection_and_null_runs) { + const std::vector null_runs = {2, 2, 3, 1}; + const std::vector selection = {0, 2, 3, 6}; + NullMap null_map = {9}; + ColumnSelectVector select_vector; + + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 8, &null_map, selection.data(), + selection.size()) + .ok()); + EXPECT_TRUE(select_vector.has_filter()); + EXPECT_EQ(select_vector.num_values(), 8); + EXPECT_EQ(select_vector.num_nulls(), 3); + EXPECT_EQ(select_vector.num_filtered(), 4); + EXPECT_EQ(null_map, (NullMap {9, 0, 1, 1, 0})); + + using ReadType = ColumnSelectVector::DataReadType; + const std::vector> expected_runs = { + {1, ReadType::CONTENT}, {1, ReadType::FILTERED_CONTENT}, + {2, ReadType::NULL_DATA}, {2, ReadType::FILTERED_CONTENT}, + {1, ReadType::CONTENT}, {1, ReadType::FILTERED_NULL}, + }; + for (const auto& [expected_length, expected_type] : expected_runs) { + ReadType type; + EXPECT_EQ(select_vector.get_next_run(&type), expected_length); + EXPECT_EQ(type, expected_type); + } + ReadType type; + EXPECT_EQ(select_vector.get_next_run(&type), 0); +} + +TEST_F(ColumnSelectVectorTest, test_dense_index_selection_uses_null_runs_directly) { + const std::vector null_runs = {2, 0, 1, 2, 1}; + NullMap null_map; + ColumnSelectVector select_vector; + + ASSERT_TRUE(select_vector.init_from_selection(null_runs, 6, &null_map, nullptr, 6).ok()); + EXPECT_FALSE(select_vector.has_filter()); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 1, 0})); + + using ReadType = ColumnSelectVector::DataReadType; + ReadType type; + EXPECT_EQ(select_vector.get_next_run(&type), 2); + EXPECT_EQ(type, ReadType::CONTENT); + EXPECT_EQ(select_vector.get_next_run(&type), 1); + EXPECT_EQ(type, ReadType::CONTENT); + EXPECT_EQ(select_vector.get_next_run(&type), 2); + EXPECT_EQ(type, ReadType::NULL_DATA); + EXPECT_EQ(select_vector.get_next_run(&type), 1); + EXPECT_EQ(type, ReadType::CONTENT); + EXPECT_EQ(select_vector.get_next_run(&type), 0); +} + +TEST_F(ColumnSelectVectorTest, test_empty_index_selection_skips_content_and_nulls) { + const std::vector null_runs = {2, 2, 3, 1}; + NullMap null_map = {7}; + ColumnSelectVector select_vector; + + ASSERT_TRUE(select_vector.init_from_selection(null_runs, 8, &null_map, nullptr, 0).ok()); + EXPECT_TRUE(select_vector.has_filter()); + EXPECT_EQ(select_vector.num_filtered(), 8); + EXPECT_EQ(null_map, (NullMap {7})); + + using ReadType = ColumnSelectVector::DataReadType; + ReadType type; + EXPECT_EQ(select_vector.get_next_run(&type), 2); + EXPECT_EQ(type, ReadType::FILTERED_CONTENT); + EXPECT_EQ(select_vector.get_next_run(&type), 2); + EXPECT_EQ(type, ReadType::FILTERED_NULL); + EXPECT_EQ(select_vector.get_next_run(&type), 3); + EXPECT_EQ(type, ReadType::FILTERED_CONTENT); + EXPECT_EQ(select_vector.get_next_run(&type), 1); + EXPECT_EQ(type, ReadType::FILTERED_NULL); + EXPECT_EQ(select_vector.get_next_run(&type), 0); +} + +TEST_F(ColumnSelectVectorTest, test_empty_null_runs_mean_all_non_null) { + const std::vector null_runs; + const std::vector selection = {1, 4}; + NullMap null_map; + ColumnSelectVector select_vector; + + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, 6, &null_map, selection.data(), + selection.size()) + .ok()); + EXPECT_EQ(null_map, (NullMap {0, 0})); + + using ReadType = ColumnSelectVector::DataReadType; + const std::vector> expected_runs = { + {1, ReadType::FILTERED_CONTENT}, {1, ReadType::CONTENT}, + {2, ReadType::FILTERED_CONTENT}, {1, ReadType::CONTENT}, + {1, ReadType::FILTERED_CONTENT}, + }; + for (const auto& [expected_length, expected_type] : expected_runs) { + ReadType type; + EXPECT_EQ(select_vector.get_next_run(&type), expected_length); + EXPECT_EQ(type, expected_type); + } +} + +TEST_F(ColumnSelectVectorTest, test_index_selection_rejects_invalid_contracts) { + ColumnSelectVector select_vector; + const std::vector null_runs = {4}; + const std::vector duplicate = {1, 1}; + const std::vector descending = {2, 1}; + const std::vector outside = {4}; + + EXPECT_FALSE(select_vector.init_from_selection(null_runs, 4, nullptr, nullptr, 2).ok()); + EXPECT_FALSE( + select_vector + .init_from_selection(null_runs, 4, nullptr, duplicate.data(), duplicate.size()) + .ok()); + EXPECT_FALSE(select_vector + .init_from_selection(null_runs, 4, nullptr, descending.data(), + descending.size()) + .ok()); + EXPECT_FALSE( + select_vector.init_from_selection(null_runs, 4, nullptr, outside.data(), outside.size()) + .ok()); + EXPECT_FALSE(select_vector.init_from_selection(null_runs, 4, nullptr, nullptr, 5).ok()); + EXPECT_FALSE( + select_vector.init_from_selection(std::vector {3}, 4, nullptr, nullptr, 4) + .ok()); + EXPECT_FALSE( + select_vector.init_from_selection(std::vector {5}, 4, nullptr, nullptr, 4) + .ok()); +} + +TEST_F(ColumnSelectVectorTest, test_reinitialization_resets_selection_mode) { + const std::vector null_runs = {4}; + const std::vector selection = {1}; + ColumnSelectVector select_vector; + ASSERT_TRUE( + select_vector + .init_from_selection(null_runs, 4, nullptr, selection.data(), selection.size()) + .ok()); + + std::vector filter_data = {1, 0, 1, 0}; + FilterMap filter_map; + ASSERT_TRUE(filter_map.init(filter_data.data(), filter_data.size(), false).ok()); + ASSERT_TRUE(select_vector.init(null_runs, 4, nullptr, &filter_map, 0).ok()); + + using ReadType = ColumnSelectVector::DataReadType; + ReadType type; + EXPECT_EQ(select_vector.get_next_run(&type), 1); + EXPECT_EQ(type, ReadType::CONTENT); + EXPECT_EQ(select_vector.get_next_run(&type), 1); + EXPECT_EQ(type, ReadType::FILTERED_CONTENT); +} + +TEST_F(ColumnSelectVectorTest, test_index_selection_exhaustive_small_batches) { + using ReadType = ColumnSelectVector::DataReadType; + for (size_t num_values = 1; num_values <= 6; ++num_values) { + const size_t bitmap_count = size_t {1} << num_values; + for (size_t null_bitmap = 0; null_bitmap < bitmap_count; ++null_bitmap) { + std::vector null_runs; + bool current_run_is_null = false; + uint16_t current_run_length = 0; + for (size_t row = 0; row < num_values; ++row) { + const bool row_is_null = ((null_bitmap >> row) & 1) != 0; + if (row_is_null == current_run_is_null) { + ++current_run_length; + continue; + } + null_runs.push_back(current_run_length); + current_run_length = 1; + current_run_is_null = row_is_null; + } + null_runs.push_back(current_run_length); + + for (size_t selection_bitmap = 0; selection_bitmap < bitmap_count; ++selection_bitmap) { + std::vector selection; + NullMap expected_null_map; + std::vector expected_actions; + for (size_t row = 0; row < num_values; ++row) { + const bool row_is_null = ((null_bitmap >> row) & 1) != 0; + const bool row_is_selected = ((selection_bitmap >> row) & 1) != 0; + if (row_is_selected) { + selection.push_back(static_cast(row)); + expected_null_map.push_back(row_is_null); + } + expected_actions.push_back( + row_is_selected + ? (row_is_null ? ReadType::NULL_DATA : ReadType::CONTENT) + : (row_is_null ? ReadType::FILTERED_NULL + : ReadType::FILTERED_CONTENT)); + } + + NullMap null_map; + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector + .init_from_selection(null_runs, num_values, &null_map, + selection.data(), selection.size()) + .ok()); + EXPECT_EQ(null_map, expected_null_map); + + std::vector actual_actions; + ReadType type; + if (selection.size() == num_values) { + while (const size_t run_length = select_vector.get_next_run(&type)) { + actual_actions.insert(actual_actions.end(), run_length, type); + } + } else { + while (const size_t run_length = select_vector.get_next_run(&type)) { + actual_actions.insert(actual_actions.end(), run_length, type); + } + } + EXPECT_EQ(actual_actions, expected_actions) + << "num_values=" << num_values << ", null_bitmap=" << null_bitmap + << ", selection_bitmap=" << selection_bitmap; + } + } + } +} + } // namespace doris diff --git a/be/test/format/parquet/parquet_thrift_test.cpp b/be/test/format/parquet/parquet_thrift_test.cpp index 0fe101db598138..d67170dca7743c 100644 --- a/be/test/format/parquet/parquet_thrift_test.cpp +++ b/be/test/format/parquet/parquet_thrift_test.cpp @@ -71,6 +71,22 @@ class ParquetThriftReaderTest : public testing::Test { void TearDown() override { TimezoneUtils::clear_timezone_caches(); } }; +TEST_F(ParquetThriftReaderTest, column_chunk_schema_is_independent_of_field_schema_lifetime) { + FieldSchema field; + field.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; + field.definition_level = 5; + field.repetition_level = 2; + field.repeated_parent_def_level = 4; + field.parquet_schema.__set_type_length(16); + + const auto chunk_schema = ParquetColumnChunkSchema::from_field_schema(field); + EXPECT_EQ(chunk_schema.physical_type, tparquet::Type::FIXED_LEN_BYTE_ARRAY); + EXPECT_EQ(chunk_schema.type_length, 16); + EXPECT_EQ(chunk_schema.max_definition_level, 5); + EXPECT_EQ(chunk_schema.max_repetition_level, 2); + EXPECT_EQ(chunk_schema.repeated_parent_definition_level, 4); +} + TEST_F(ParquetThriftReaderTest, normal) { auto local_fs = io::global_local_filesystem(); io::FileReaderSPtr reader; diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 36b7cebdaa9cb9..5de3efbe3e7b7c 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -396,9 +396,20 @@ struct ScalarColumnReaderTestAccess { reader->_row_group_rows_read = rows; } + static const ParquetLeafReader* leaf_reader_address(ScalarColumnReader& reader) { + return &reader.leaf_reader(); + } + + static size_t dictionary_binary_capacity(const ScalarColumnReader& reader) { + return reader._dictionary_binary_values.capacity(); + } + + static size_t dictionary_binary_size(const ScalarColumnReader& reader) { + return reader._dictionary_binary_values.size(); + } + static Status append_dictionary_filtered_values( - const ScalarColumnReader& reader, - const std::vector>& chunks, + ScalarColumnReader& reader, const std::vector>& chunks, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { return reader.append_dictionary_filtered_values(chunks, dictionary_filter, column, @@ -495,6 +506,34 @@ TEST(ParquetScalarColumnReaderTest, DictionaryIndexOutsideFilterIsCorruption) { EXPECT_NE(status.to_string().find("Invalid parquet dictionary index 1"), std::string::npos); } +TEST(ParquetScalarColumnReaderTest, LeafReaderAndDictionaryScratchArePersistent) { + ScalarColumnReader reader(string_schema("persistent_leaf"), nullptr); + const auto* leaf_reader = ScalarColumnReaderTestAccess::leaf_reader_address(reader); + + MutableColumnPtr column = ColumnString::create(); + IColumn::Filter row_filter; + int64_t matched_rows = 0; + bool used_filter = false; + const std::vector> chunks = { + dictionary_array({0, 1}, {"first", "second"})}; + ASSERT_TRUE(ScalarColumnReaderTestAccess::append_dictionary_filtered_values( + reader, chunks, IColumn::Filter {1, 1}, column, &row_filter, &matched_rows, + &used_filter) + .ok()); + const auto retained_capacity = ScalarColumnReaderTestAccess::dictionary_binary_capacity(reader); + EXPECT_GE(retained_capacity, 2); + EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_size(reader), 0); + + row_filter.clear(); + ASSERT_TRUE(ScalarColumnReaderTestAccess::append_dictionary_filtered_values( + reader, chunks, IColumn::Filter {1, 0}, column, &row_filter, &matched_rows, + &used_filter) + .ok()); + EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_capacity(reader), retained_capacity); + EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_size(reader), 0); + EXPECT_EQ(ScalarColumnReaderTestAccess::leaf_reader_address(reader), leaf_reader); +} + } // namespace TEST(SelectionVectorTest, IdentitySelectionToRanges) { @@ -520,6 +559,45 @@ TEST(SelectionVectorTest, ExternalBufferSelectionToRanges) { EXPECT_TRUE(selection.verify(std::size(indices), 8).ok()); } +TEST(SelectionVectorTest, OutputRangesReuseCapacity) { + SelectionVector::Index indices[] = {1, 2, 5}; + SelectionVector selection(indices, std::size(indices)); + std::vector ranges; + ranges.reserve(8); + const auto retained_capacity = ranges.capacity(); + + selection_to_ranges(selection, std::size(indices), &ranges); + ASSERT_EQ(ranges.size(), 2); + EXPECT_EQ(ranges.capacity(), retained_capacity); + selection_to_ranges(selection, 1, &ranges); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges.capacity(), retained_capacity); +} + +TEST(ParquetNestedScalarBatchTest, ResetRetainsScratchCapacityAndColumn) { + ParquetNestedScalarBatch batch; + batch.def_levels.reserve(16); + batch.rep_levels.reserve(16); + batch.value_indices.reserve(16); + batch.def_levels.push_back(1); + batch.rep_levels.push_back(0); + batch.value_indices.push_back(0); + batch.values_column = ColumnInt64::create(); + batch.values_column->insert_default(); + const auto* values_column = batch.values_column.get(); + + batch.reset(); + + EXPECT_TRUE(batch.def_levels.empty()); + EXPECT_TRUE(batch.rep_levels.empty()); + EXPECT_TRUE(batch.value_indices.empty()); + EXPECT_GE(batch.def_levels.capacity(), 16); + EXPECT_GE(batch.rep_levels.capacity(), 16); + EXPECT_GE(batch.value_indices.capacity(), 16); + EXPECT_EQ(batch.values_column.get(), values_column); + EXPECT_TRUE(batch.values_column->empty()); +} + TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { SelectionVector selection(2); EXPECT_FALSE(selection.verify(3, 3).ok()); diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 63f63819670dd2..c88ede9974dc95 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -25,8 +25,9 @@ predicate columns for surviving ranges, and finally defer output-column reads un - **Layered caches:** File-block cache, Parquet page cache, condition-result cache, and merged small I/O solve different problems and are not interchangeable. -**Scope:** This document focuses on the FileScannerV2 Parquet Reader design and core pipeline. It -does not cover Arrow decoder internals, complex-type reconstruction, or expression implementation. +**Scope:** This document covers the FileScannerV2 Parquet Reader pipeline, the native page/decode +kernel, selection-aware materialization, and complex-type reconstruction contracts. Expression +implementation details remain outside its scope. ## 2. Overall Architecture @@ -40,9 +41,10 @@ flowchart TB B --> C[TableReader
Schema Mapping, Partition/Default Values, Predicate Localization] C --> D[ParquetReader
Footer/Schema and Row Group Scan Planning] D --> E[ParquetScanScheduler
Row Group Lifecycle and Batch Reads] - E --> F[ParquetColumnReader
Page Skipping, Decompression, Decoding, Materialization] - F --> G[ParquetFileContext / Arrow RandomAccessFile
Page Cache, MergeRange, Prefetch] - G --> H[Doris FileReader / FileCache / Remote FS] + E --> F[ParquetColumnReader
Persistent Column State and Complex Reconstruction] + F --> G[Native ColumnChunk / Page / Encoding Decoders
Selection-aware Doris Materialization] + G --> H[ParquetFileContext / Stream Adapter
Page Cache, MergeRange, Prefetch] + H --> I[Doris FileReader / FileCache / Remote FS] ``` | Layer | Core responsibilities | Responsibilities intentionally excluded | @@ -58,6 +60,64 @@ flowchart TB > layer can use footer, page index, dictionary, and other format knowledge while upper layers retain > uniform scan semantics. +### 2.1 Native Reader Target and Migration State + +The target execution path does not use Arrow builders or `ReadRecords` to decode data pages. Doris +owns the Column Chunk, page, level, encoding, selection, conversion, and materialization state, and +writes directly into Doris columns. This follows the same broad separation used by DuckDB: metadata +planning is distinct from a persistent per-column reader, and page/encoding decoders expose narrow +cursor-based contracts rather than an Arrow array as an intermediate result. + +The migration is deliberately incremental because the existing v1 native kernel already contains +mature Parquet page and encoding support: + +| Stage | State and boundary | +| --- | --- | +| Native encoding kernel | Reuse and extract the Doris v1 decoders behind a schema-independent physical Column Chunk contract. Selection-aware scalar decoding and reusable string scratch are the first vertical slice. | +| Native column reader | Make the leaf reader and its SerDe, null map, selection ranges, binary values, level buffers, and conversion scratch persistent for a Row Group. Add direct Doris materialization paths. | +| Complex reconstruction | Build one shared Dremel level plan per requested parent-row range and use it for STRUCT/ARRAY/MAP siblings, offsets, and null maps. | +| Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | +| Compatibility removal | Remove Arrow data-read adapters only after type/encoding/page/writer compatibility and performance gates pass. Keep an explicit fallback during migration; never silently select a partially supported native path. | + +At the current migration boundary, Arrow may still be present in the v2 footer/schema, planning, or +I/O adapter path. That does not make Arrow arrays part of the native decoder contract. Documentation +and Profile names must distinguish the native data-page path from remaining metadata/adaptation +work so an intermediate patch is not mistaken for complete Arrow removal. + +All production integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the +correctness and performance control. Extracting a narrow shared decoder primitive is permitted when +it preserves the legacy API and behavior, but the migration does not route v1 reads through a new +v2 path. This separation makes differential testing meaningful and prevents a v2 performance +experiment from regressing the established reader. + +### 2.2 Native Interface Ownership + +```mermaid +flowchart LR + A[Native Footer / Physical Schema] --> B[RowGroupReadPlan] + B --> C[Persistent ParquetColumnReader] + C --> D[Persistent ColumnChunkReader] + D --> E[Page Reader + Level Decoders] + E --> F[Encoding Decoder] + F --> G[ColumnSelectVector] + G --> H[Doris MutableColumn] + C --> I[Shared Complex Level Plan] + I --> G +``` + +- **Physical schema contract:** Immutable physical type, fixed length, maximum definition and + repetition levels, and repeated-parent definition threshold. It owns no Arrow descriptor and + borrows no temporary table-schema object. +- **Persistent column state:** Page/decompress buffers, dictionary decoder, SerDe/conversion state, + null maps, selection ranges, binary-value references, and builder capacity live with the column + reader and are logically reset rather than recreated for each batch. +- **Decoder contract:** Consume a known number of logical level entries and encoded payload values, + then materialize only selected rows. Decoders do not decide table projection, predicate meaning, + or parent complex offsets. +- **Complex plan contract:** Definition/repetition levels are parsed once into parent-row boundaries + and child-presence/null decisions. All children consume the same plan so their offsets and null + maps cannot drift. + ## 3. From File Open to Scan Plan After a reader receives a Split, it opens the file and builds the scan plan. This phase determines @@ -75,7 +135,7 @@ sequenceDiagram FS->>TR: prepare/open split TR->>TR: Map schema and localize predicates TR->>PR: FileScanRequest - PR->>FC: Open FileReader + PR->>FC: Open FileReader / native stream adapter FC->>META: Read footer and schema META-->>PR: Row Group / Column Chunk metadata PR->>PLAN: Candidate Row Groups and local predicates @@ -94,8 +154,10 @@ sequenceDiagram delete conjuncts, and local column-position mappings. - **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` produced by page-index pruning, and the `page_skip_plan` for each leaf column. -- **ParquetFileContext:** Adapts Doris FileReader to Arrow RandomAccessFile and owns Page Cache, - FileCache prefetch, and MergeRange routing. +- **ParquetFileContext:** Adapts Doris FileReader to the active metadata/data stream interface and + owns Page Cache, FileCache prefetch, and MergeRange routing. During migration this may still + expose an Arrow adapter for metadata consumers, but native page decoding reads the same stable + Doris byte ranges without producing Arrow arrays. > Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the > candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data @@ -279,6 +341,175 @@ flowchart LR > output columns must decode and copy. This is the main benefit of lazy materialization in a > columnar format. +### 7.1 Selection and Cursor Contract + +The native decoder receives a sorted selection over logical rows. Logical positions include nulls; +they are not offsets into the encoded non-null payload. Definition levels and selection are merged +as two ordered streams into four run types: + +| Run | Consume logical level entries | Consume encoded payload | Append output | +| --- | --- | --- | --- | +| Selected non-null (`CONTENT`) | Yes | Yes | Value | +| Selected null (`NULL_DATA`) | Yes | No | Default plus null-map bit | +| Filtered non-null (`FILTERED_CONTENT`) | Yes | Yes or decoder-native skip | No | +| Filtered null (`FILTERED_NULL`) | Yes | No | No | + +The contract keeps three counts explicit: logical level entries consumed, encoded non-null values +consumed, and Doris output values appended. The page reader may cross page boundaries while +satisfying one batch, but all three counts must be aligned when it returns. A dense identity +selection, an empty selection, and an arbitrary fragmented selection use the same decoder API. +Selection inputs are borrowed only for the duration of the decode call. + +For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes +definition-level runs, builds the four-way selection plan without an O(batch rows) action array, +and dispatches the plan to the active encoding decoder. A filtered non-null value must still advance +the encoding state even when it is not copied. This is the central invariant that prevents the next +batch from decoding shifted values. + +### 7.2 Page and Encoding Kernel + +The native page reader parses Page V1 and Page V2 headers, obtains level streams and payload bytes, +decompresses only the required region, and installs a decoder selected from physical type plus data +encoding. Page V1 and V2 differ in where levels are stored and which bytes are compressed; after +that parsing step both feed the same level and value-decoder contracts. + +| Encoding family | Native responsibility | +| --- | --- | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, decode/skip IDs by selection, and reject invalid IDs | +| RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | +| DELTA_BINARY_PACKED | Preserve block/mini-block state while selected and filtered values share one payload cursor | +| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, including skipped values | +| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value and binary scratch state | +| BYTE_STREAM_SPLIT | Reassemble primitive lanes and apply selection without an Arrow intermediate | + +Unsupported physical-type/encoding combinations return an explicit error or use a declared +compatibility fallback. They never produce a plausible result through a decoder selected only by +logical Doris type. Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated +payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. + +### 7.3 Direct Materialization and Scratch Reuse + +Decoders write directly into Doris mutable columns whenever physical and target layouts are +compatible. Fixed-width primitives reserve destination capacity once and append contiguous selected +runs. String-like decoders gather selected `StringRef` values into persistent scratch and perform +one batched append, so fragmented predicates do not cause one destination growth or copy call per +run. Scratch stores references only while the backing page/dictionary buffer is stable. + +Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian +two's-complement values with correct sign extension, and apply precision/scale conversion exactly +once. Date, timestamp, INT96, unsigned annotations, CHAR/VARCHAR, and timezone conversions retain +the same semantic checks as the general conversion path. A fast path is enabled only when those +checks prove the result is equivalent. + +The persistent leaf reader owns reusable SerDe/conversion objects, null map, selection ranges, +definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris +column/builder capacity. Logical sizes are reset at batch boundaries while capacity is retained. +During migration, any remaining Arrow builder fallback follows the same lifetime and capacity-reuse +rule; it must not be constructed once per `ReadRecords` call. + +### 7.4 Complex Types and Shared Level Plans + +Repeated Parquet leaves cannot interpret a requested parent-row count as a leaf-value count. One +parent row may contain zero, one, or many level entries, and its final entry can reside on the next +page. The complex reader therefore builds a shared level plan with: + +- parent-row start/end boundaries derived from repetition levels; +- ancestor-null, collection-null, empty-collection, element-null, and present-value decisions from + definition thresholds; +- child payload positions and selected parent rows; +- cross-page continuation state for an unfinished parent row. + +ARRAY and MAP offsets and null maps are derived from that plan. STRUCT children reuse the plan from +a representative present leaf and advance in parent-row lockstep; a missing or fully projected-out +child is materialized from the same parent count. MAP key/value readers must produce identical +entry counts, and Parquet's non-null key requirement is validated rather than repaired. + +Level scratch is sized by decoded level entries, not by the 16-bit parent batch cap. Long repeated +rows and long null/non-null runs are split into representable internal runs without introducing a +new row boundary. Tests cover null ancestors, empty collections, null elements/values, nested +STRUCT-in-ARRAY and ARRAY-in-STRUCT shapes, sibling page misalignment, and rows spanning pages and +batches. + +#### Complex-reader interface and materialization cost + +The Arrow migration adapter currently exposes four stateful operations: +`load_nested_batch()`, `load_nested_levels_batch()`, `build_nested_column()`, and +`consume_nested_column()`. This split made shape-only reads possible while Arrow still owned value +decoding, but it is not the target native interface. It has three measurable costs: + +1. callers must preserve an implicit load-before-build/consume phase and multiple nested cursors; +2. ARRAY/MAP/STRUCT layers can rescan the same def/rep span to derive parent boundaries, child + counts, null maps, and alignment; +3. a materialized leaf can copy levels, build a level-to-payload index, build a temporary null map, + and only then convert the payload to a Doris column. + +Doris v1 is simpler at the public boundary: a child `read_column_data()` call owns physical decode +and exposes its persistent def/rep buffers to the collection reader. It nevertheless repeats some +level interpretation in collection helpers, so v1 is a compatibility/performance baseline rather +than the final abstraction. DuckDB uses a single `Read(input, vector)` operation. Its LIST reader +reads a reusable child vector plus def/rep buffers, emits list entries in the same traversal, and +keeps overflow for the next vector; its STRUCT reader invokes children directly into output vectors +and verifies their row counts. See DuckDB's official +[LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), +[STRUCT reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/struct_column_reader.cpp), +and [base column reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/column_reader.cpp). + +The native v2 boundary therefore uses one operation conceptually equivalent to: + +```text +read_nested(request { parent_rows, selection, VALUES | LEVELS_ONLY }, output) + -> result { parent_rows, shared_level_plan, payload_counts } +``` + +`VALUES` decodes selected payload and materializes directly into the supplied Doris child columns; +`LEVELS_ONLY` advances identical level/page cursors without materializing payload. Both modes build +the same parent boundaries and validation decisions. ARRAY, MAP, and STRUCT consume a shared plan +rather than calling separate public build/consume phases. For example, levels representing +`[["a", "b"], NULL, []]` produce parent boundaries `[0, 2, 3, 4]`, entry counts `[2, 0, 0]`, and +parent nulls `[0, 1, 0]` in one traversal; string payload ordinals are `[0, 1]`. MAP uses the key +leaf as the entry-shape owner and validates the value leaf against the same entry plan. STRUCT uses +one representative leaf for parent validity and only checks sibling alignment while decoding each +child. + +During migration, the four Arrow-facing methods remain a compatibility facade. Their leaf reader, +SerDe, binary/null/level scratch, selection ranges, nested batches, parent nulls, entry counts, and +child-column handles must be persistent so the facade does not add per-batch allocation churn. +New native decoder code must not depend on this phase ordering or reproduce the temporary +level-to-payload index when it can stream payload positions directly from the shared plan. + +### 7.5 Index Coordinate Domains and Composition + +Index correctness depends on keeping its coordinate systems separate: + +| Identity | Scope | Must not be confused with | +| --- | --- | --- | +| Table/local column ID | Current file request and Doris block | Physical Parquet leaf ordinal | +| Physical leaf-column ID | Footer Row Group Column Chunk array | Logical parent STRUCT/ARRAY/MAP ordinal | +| Row Group ID | File metadata | Split ordinal or batch ordinal | +| Data-page ordinal | One Column Chunk, excluding dictionary pages | PageHeader sequence including dictionary page | +| OffsetIndex row ordinal | Logical row inside a Row Group | Encoded non-null value position | +| Selection index | Logical row inside the current batch, nulls included | Dictionary ID or compact output position | +| Dictionary-entry ID | One Column Chunk dictionary | Row ordinal, global dictionary ID, or the next Row Group's dictionary | + +Row Group statistics, dictionary pruning, Bloom, ColumnIndex/OffsetIndex, page skip plans, cache +ranges, row selection, and lazy predicate/output readers are composed in that order. Each stage may +only remove candidates already expressed in the same Row Group logical-row domain. Page ordinals +from ColumnIndex and OffsetIndex are validated together before they become `selected_ranges` and a +per-leaf physical page-skip plan. + +Dictionary row filtering starts only when metadata proves every data page in the Column Chunk uses +a dictionary encoding. The predicate is evaluated against the current dictionary to produce an +entry bitmap; decoded IDs are checked against both dictionary length and bitmap length. A mixed +dictionary/plain transition falls back before consuming data. Once selected dictionary reading has +advanced a page cursor, loss of dictionary output is corruption rather than a retry through another +path with shifted state. + +Missing optional indexes or unsupported predicate/type combinations retain rows. Malformed +offsets, inconsistent page counts, out-of-range dictionary IDs, overlapping/unsorted invalid ranges, +or an impossible cursor relationship are reported as corruption. Cache hits and misses do not +change any page, level, value, or dictionary cursor. + ## 8. Supported Indexes and Their Boundaries V2 uses native Parquet metadata and encoding information. It does not construct Doris-internal @@ -315,12 +546,12 @@ flowchart TD ## 9. Cache and I/O Optimization -Parquet V2 has four complementary cache and I/O paths: cache remote file blocks, cache serialized -Parquet ranges, cache predicate results, and merge small random reads. +Parquet V2 has five complementary cache and I/O paths: cache footer/parsed metadata, cache remote +file blocks, cache serialized Parquet ranges, cache predicate results, and merge small random reads. ```mermaid flowchart TB - A[Parquet Column Reader ReadAt] --> B{Parquet Page Cache Hit?} + A[Parquet Column Reader Range Read] --> B{Parquet Page Cache Hit?} B -- "Yes" --> C[Return Cached Serialized Range Bytes] B -- "No" --> D{MergeRange Active?} D -- "Yes" --> E[MergeRangeFileReader
Merge Adjacent Small I/O] @@ -334,11 +565,35 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | +| Footer metadata cache | Serialized footer bytes and immutable parsed native metadata | Stable file identity matching v1: path plus size and trustworthy modification/version information, with schema-affecting options in the parsed-object key | Avoid repeated footer I/O, Thrift parsing, and schema construction across scans | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | | MergeRangeFileReader | Not a cache; merges small ranges into larger slices | Installed temporarily for projected chunks of the current Row Group | Reduce remote small-I/O count and request overhead | +### Footer Cache Parity with V1 + +V2 uses the same cacheability and invalidation policy as v1 rather than introducing a second notion +of file identity. A hit returns immutable metadata that can be shared by readers; mutable Row Group, +selection, decoder, and scratch state remains per reader. Path-only keys are insufficient. When a +trustworthy version identity is unavailable, the footer is read and parsed without publishing a +reusable entry. Parse failures, short files, encrypted/unsupported metadata, and schema-affecting +option changes cannot populate or reuse a successful entry. + +During migration, serialized-footer caching and parsed-native-metadata caching may be enabled in +separate steps, but both use the same identity and lifetime rules. An Arrow-parsed metadata object +must not be treated as the native cache value or leak an Arrow lifetime into the native decoder. + +### Page Cache Parity with V1 + +The native path uses v1's page-cache semantics as its minimum contract: the same stable file +identity, Column Chunk/page byte ranges, compressed versus decompressed entry distinction, checksum +and decompression ownership, subrange coverage, invalidation, admission, and fallback I/O rules. +Mutable decoder and dictionary state is never cached. A hit returns immutable bytes and advances the +reader exactly as the corresponding base read would; a miss performs normal I/O before optional +admission. An alternative policy is accepted only with correctness coverage plus warm/cold, +selective/dense, local/remote benchmarks showing it is no worse than v1. + ### Why Page Cache registers only surviving chunks The footer is read before Row Group planning and before Page Cache ranges are registered, so @@ -350,7 +605,8 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. - When average projected chunks are small and the reader is not in-memory, install - MergeRangeFileReader so subsequent Arrow `ReadAt` calls actually use merged reads. + MergeRangeFileReader so subsequent native range reads or transitional Arrow `ReadAt` calls use + merged reads. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. @@ -396,6 +652,14 @@ flowchart LR E --> F[Bound by batch_size and Selected Range] ``` +Changing the requested row cap changes only the amount of work performed by the persistent reader. +It does not recreate Column Readers, Column Chunk readers, dictionaries, SerDe/conversion objects, +builders, or scratch. The probe is charged as a normal batch and estimates completed Doris rows and +bytes after table-level materialization, matching v1's measurement point. Empty or highly filtered +probes do not permanently collapse the batch size; the scheduler retains enough evidence before +updating its estimate and respects page/range boundaries without turning each fragment into a new +reader lifecycle. + ### 10.3 Aggregate Pushdown When TableReader proves that no filter or delete semantics can change the result, COUNT / MIN / MAX @@ -469,6 +733,10 @@ flowchart TD | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | | Condition Cache | How many rows were skipped early after a cache hit? | +| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? | +| Batch fragmentation | How does `TotalBatches` divide into adaptive probes, dense, selected, empty, page-crossing, and nested/fragmented batches? | +| Index decisions | How often were statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skips attempted, accepted, conservatively rejected, or rejected as corrupt? | +| Cache lifecycle | For footer/page/file/condition caches, what were request, hit, miss, bypass, admission/write, byte, wait, and underlying-I/O counts using v1-compatible meanings? | > Interpret pruning ratios in the context of write layout. Unsorted data produces wide min/max > ranges, so Row Group/Page pruning may be ineffective even when the reader and indexes work @@ -476,14 +744,16 @@ flowchart TD ## 13. Summary -The FileScannerV2 Parquet scan pipeline has three primary threads: +The FileScannerV2 Parquet scan pipeline has four primary threads: 1. **Semantic thread:** TableReader maps table schema and predicates into stable file-local semantics, preserving schema evolution, partition columns, and missing columns. 2. **Pruning thread:** Split → Row Group → Page → Row progressively applies Runtime Filters, Statistics, Dictionary, Bloom, Page Index, and actual-value filters. -3. **I/O thread:** Predicate-first reads, SelectionVector, lazy materialization, adaptive batches, - FileCache/Page Cache/Condition Cache, and MergeRange reduce read amplification together. +3. **Decode thread:** Persistent native page/encoding readers merge Dremel levels with Selection, + reuse scratch, reconstruct complex values from shared plans, and materialize directly to Doris. +4. **I/O thread:** Predicate-first reads, adaptive batches, Footer/File/Page/Condition caches, and + MergeRange reduce read amplification together. ```mermaid flowchart LR From f49708353686ae3eebc3a50f5d090d971b478d1e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 10:53:19 +0800 Subject: [PATCH 02/34] [fix](be) Bound Parquet v2 binary batch lifetime ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: FileScannerV2 retained Arrow binary and dictionary chunks in persistent Parquet reader scratch after synchronous Doris materialization. Large binary batches could therefore stay alive until a later read and overlap the next Arrow builder allocation. Release batch payload ownership on every success and error path while retaining vector capacity, clear borrowed StringRef scratch before its owners are released, and add unit coverage for ownership release and capacity reuse. Also remove the previous decoder changes under the legacy format directory so the v1 implementation remains unchanged; future native decoder work is constrained to format_v2. This lifecycle fix addresses the review finding but does not claim to eliminate the separate complex-column temporary materialization overhead. That v2-only decoder/materializer redesign remains distinct from COUNT(nullable_col) pushdown. ### Release note None ### Check List (For Author) - Test: Unit Test - Remote ASAN BE build - 16 focused Parquet v2 BE unit tests - Remote clang-format check - Remote clang-tidy attempted; blocked by existing diagnostics and the remote toolchain missing stddef.h during analysis - Behavior changed: No; internal buffer ownership is released earlier - Does this need documentation: Yes; updated the FileScannerV2 Parquet design document and format_v2 review guide --- .../parquet/byte_array_dict_decoder.cpp | 15 +- .../format/parquet/byte_array_dict_decoder.h | 3 - .../parquet/byte_array_plain_decoder.cpp | 25 +- .../format/parquet/byte_array_plain_decoder.h | 7 - .../format/parquet/delta_bit_pack_decoder.h | 19 +- be/src/format/parquet/parquet_common.cpp | 178 --------------- be/src/format/parquet/parquet_common.h | 65 +----- .../parquet/vparquet_column_chunk_reader.cpp | 87 +------ .../parquet/vparquet_column_chunk_reader.h | 55 +---- be/src/format_v2/AGENTS.md | 13 +- .../parquet/reader/parquet_leaf_reader.cpp | 37 ++- .../parquet/reader/parquet_leaf_reader.h | 5 + .../parquet/reader/scalar_column_reader.cpp | 17 +- .../parquet/byte_array_dict_decoder_test.cpp | 27 --- .../parquet/byte_array_plain_decoder_test.cpp | 39 ---- .../byte_stream_split_decoder_test.cpp | 32 --- .../parquet/delta_bit_pack_decoder_test.cpp | 26 --- .../parquet/fix_length_plain_decoder_test.cpp | 29 --- .../format/parquet/parquet_common_test.cpp | 216 ------------------ .../format/parquet/parquet_thrift_test.cpp | 16 -- .../parquet/parquet_leaf_reader_test.cpp | 29 +++ .../parquet/parquet_reader_control_test.cpp | 1 + docs/file-scanner-v2-parquet-scan-design.md | 43 ++-- 23 files changed, 151 insertions(+), 833 deletions(-) diff --git a/be/src/format/parquet/byte_array_dict_decoder.cpp b/be/src/format/parquet/byte_array_dict_decoder.cpp index 1bc1a0b5b4acb5..c7ca433223ed7a 100644 --- a/be/src/format/parquet/byte_array_dict_decoder.cpp +++ b/be/src/format/parquet/byte_array_dict_decoder.cpp @@ -145,20 +145,22 @@ Status ByteArrayDictDecoder::_decode_values(MutableColumnPtr& doris_column, Data } size_t dict_index = 0; - _selected_values.clear(); - _selected_values.reserve(select_vector.num_values() - select_vector.num_filtered()); ColumnSelectVector::DataReadType read_type; while (size_t run_length = select_vector.get_next_run(&read_type)) { switch (read_type) { case ColumnSelectVector::CONTENT: { + DorisVector string_values; + string_values.reserve(run_length); for (size_t i = 0; i < run_length; ++i) { - _selected_values.emplace_back(_dict_items[_indexes[dict_index++]]); + string_values.emplace_back(_dict_items[_indexes[dict_index++]]); } + doris_column->insert_many_strings_overflow(string_values.data(), run_length, + _max_value_length); break; } case ColumnSelectVector::NULL_DATA: { - _selected_values.insert(_selected_values.end(), run_length, StringRef("", 0)); + doris_column->insert_many_defaults(run_length); break; } case ColumnSelectVector::FILTERED_CONTENT: { @@ -171,11 +173,6 @@ Status ByteArrayDictDecoder::_decode_values(MutableColumnPtr& doris_column, Data } } } - DCHECK_EQ(_selected_values.size(), select_vector.num_values() - select_vector.num_filtered()); - if (!_selected_values.empty()) { - doris_column->insert_many_strings_overflow(_selected_values.data(), _selected_values.size(), - _max_value_length); - } return Status::OK(); } diff --git a/be/src/format/parquet/byte_array_dict_decoder.h b/be/src/format/parquet/byte_array_dict_decoder.h index 91fbc14cd1f161..ff79b4ba94fe83 100644 --- a/be/src/format/parquet/byte_array_dict_decoder.h +++ b/be/src/format/parquet/byte_array_dict_decoder.h @@ -60,9 +60,6 @@ class ByteArrayDictDecoder final : public BaseDictDecoder { // For dictionary encoding DorisVector _dict_items; DorisVector _dict_data; - // Selected dictionary values are gathered once and appended in one ColumnString resize/copy. - // The capacity is retained across pages and batches. - DorisVector _selected_values; size_t _max_value_length; }; diff --git a/be/src/format/parquet/byte_array_plain_decoder.cpp b/be/src/format/parquet/byte_array_plain_decoder.cpp index 343518b5d23264..d9048c5549cc44 100644 --- a/be/src/format/parquet/byte_array_plain_decoder.cpp +++ b/be/src/format/parquet/byte_array_plain_decoder.cpp @@ -18,6 +18,7 @@ #include "format/parquet/byte_array_plain_decoder.h" #include +#include #include "core/column/column.h" #include "core/data_type/data_type_nullable.h" @@ -61,29 +62,37 @@ template Status ByteArrayPlainDecoder::_decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector, bool is_dict_filter) { - _selected_values.clear(); - _selected_values.reserve(select_vector.num_values() - select_vector.num_filtered()); ColumnSelectVector::DataReadType read_type; while (size_t run_length = select_vector.get_next_run(&read_type)) { switch (read_type) { case ColumnSelectVector::CONTENT: { + std::vector string_values; + string_values.reserve(run_length); for (size_t i = 0; i < run_length; ++i) { uint32_t length = 0; RETURN_IF_ERROR(read_length(_data, &_offset, &length)); if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { return Status::IOError("Can't read enough bytes in plain decoder"); } - _selected_values.emplace_back(_data->data + _offset, length); + string_values.emplace_back(_data->data + _offset, length); _offset += length; } + doris_column->insert_many_strings(&string_values[0], run_length); break; } case ColumnSelectVector::NULL_DATA: { - _selected_values.insert(_selected_values.end(), run_length, StringRef("", 0)); + doris_column->insert_many_defaults(run_length); break; } case ColumnSelectVector::FILTERED_CONTENT: { - RETURN_IF_ERROR(skip_values(run_length)); + for (int i = 0; i < run_length; ++i) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in plain decoder"); + } + _offset += length; + } break; } case ColumnSelectVector::FILTERED_NULL: { @@ -92,12 +101,6 @@ Status ByteArrayPlainDecoder::_decode_values(MutableColumnPtr& doris_column, Dat } } } - DCHECK_EQ(_selected_values.size(), select_vector.num_values() - select_vector.num_filtered()); - if (!_selected_values.empty()) { - // ColumnString calculates the aggregate byte length before copying, so this call grows the - // chars buffer and offsets exactly once for the whole selected batch. - doris_column->insert_many_strings(_selected_values.data(), _selected_values.size()); - } return Status::OK(); } diff --git a/be/src/format/parquet/byte_array_plain_decoder.h b/be/src/format/parquet/byte_array_plain_decoder.h index fd3f7d3fef52b5..192824a2aab7a3 100644 --- a/be/src/format/parquet/byte_array_plain_decoder.h +++ b/be/src/format/parquet/byte_array_plain_decoder.h @@ -21,12 +21,10 @@ #include #include -#include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" #include "core/data_type/data_type.h" -#include "core/string_ref.h" #include "core/types.h" #include "format/format_common.h" #include "format/parquet/decoder.h" @@ -54,11 +52,6 @@ class ByteArrayPlainDecoder final : public Decoder { ColumnSelectVector& select_vector, bool is_dict_filter); Status skip_values(size_t num_values) override; - -private: - // References point into the current page buffer and are valid until the batch is appended. - // Keeping the vector on the decoder reuses its capacity across pages and batches. - std::vector _selected_values; }; } // namespace doris diff --git a/be/src/format/parquet/delta_bit_pack_decoder.h b/be/src/format/parquet/delta_bit_pack_decoder.h index e32cadb86be941..6257e4f214a182 100644 --- a/be/src/format/parquet/delta_bit_pack_decoder.h +++ b/be/src/format/parquet/delta_bit_pack_decoder.h @@ -50,23 +50,23 @@ class DeltaDecoder : public Decoder { template Status decode_byte_array(const std::vector& decoded_vals, MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector) { - _selected_string_values.clear(); - _selected_string_values.reserve(select_vector.num_values() - select_vector.num_filtered()); ColumnSelectVector::DataReadType read_type; int value_idx = 0; while (size_t run_length = select_vector.get_next_run(&read_type)) { switch (read_type) { case ColumnSelectVector::CONTENT: { + std::vector string_values; + string_values.reserve(run_length); for (size_t i = 0; i < run_length; ++i) { size_t length = decoded_vals[value_idx].size; - _selected_string_values.emplace_back(decoded_vals[value_idx].data, length); + string_values.emplace_back(decoded_vals[value_idx].data, length); value_idx++; } + doris_column->insert_many_strings(&string_values[0], run_length); break; } case ColumnSelectVector::NULL_DATA: { - _selected_string_values.insert(_selected_string_values.end(), run_length, - StringRef("", 0)); + doris_column->insert_many_defaults(run_length); break; } case ColumnSelectVector::FILTERED_CONTENT: { @@ -79,12 +79,6 @@ class DeltaDecoder : public Decoder { } } } - DCHECK_EQ(_selected_string_values.size(), - select_vector.num_values() - select_vector.num_filtered()); - if (!_selected_string_values.empty()) { - doris_column->insert_many_strings(_selected_string_values.data(), - _selected_string_values.size()); - } return Status::OK(); } @@ -144,9 +138,6 @@ class DeltaDecoder : public Decoder { } // Convert decoded value to doris type value. std::unique_ptr _type_converted_decoder; - // Values reference decoder-owned page scratch. Retaining this vector avoids one allocation per - // CONTENT run and lets ColumnString resize once for the whole selected batch. - std::vector _selected_string_values; }; /** diff --git a/be/src/format/parquet/parquet_common.cpp b/be/src/format/parquet/parquet_common.cpp index 743f4b393154f4..26ec8583b7753a 100644 --- a/be/src/format/parquet/parquet_common.cpp +++ b/be/src/format/parquet/parquet_common.cpp @@ -19,8 +19,6 @@ #include -#include - #include "common/cast_set.h" #include "core/types.h" #include "util/simd/bits.h" @@ -122,10 +120,6 @@ Status FilterMap::generate_nested_filter_map(const std::vector& rep_lev Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, const std::unordered_set* skipped_indices) { - _uses_index_selection = false; - _selection_indices = nullptr; - _selected_count = 0; - _run_length_null_map = &run_length_null_map; _num_values = num_values; _num_nulls = 0; _read_index = 0; @@ -224,178 +218,6 @@ Status ColumnSelectVector::init(const std::vector& run_length_null_map return Status::OK(); } -Status ColumnSelectVector::_validate_null_runs(const std::vector& run_length_null_map, - size_t num_values) const { - if (run_length_null_map.empty()) { - return Status::OK(); - } - - size_t run_length_sum = 0; - for (const auto run_length : run_length_null_map) { - run_length_sum += run_length; - if (run_length_sum > num_values) { - return Status::InvalidArgument( - fmt::format("Parquet null runs describe {} rows, which exceeds num_values {}", - run_length_sum, num_values)); - } - } - if (run_length_sum != num_values) { - return Status::InvalidArgument(fmt::format( - "Parquet null runs describe {} rows, expected {}", run_length_sum, num_values)); - } - return Status::OK(); -} - -Status ColumnSelectVector::init_from_selection(const std::vector& run_length_null_map, - size_t num_values, NullMap* const null_map, - const uint16_t* selected_indices, - size_t selected_count) { - RETURN_IF_ERROR(_validate_null_runs(run_length_null_map, num_values)); - if (selected_count > num_values) { - return Status::InvalidArgument( - fmt::format("Parquet selection contains {} rows, which exceeds num_values {}", - selected_count, num_values)); - } - if (selected_indices == nullptr && selected_count != 0 && selected_count != num_values) { - return Status::InvalidArgument( - "A partial Parquet selection must provide explicit row indices"); - } - if (selected_indices != nullptr) { - for (size_t i = 0; i < selected_count; ++i) { - if (selected_indices[i] >= num_values) { - return Status::InvalidArgument( - fmt::format("Parquet selection index {} is outside [0, {})", - selected_indices[i], num_values)); - } - if (i != 0 && selected_indices[i - 1] >= selected_indices[i]) { - return Status::InvalidArgument( - "Parquet selection indices must be strictly increasing"); - } - } - } - - _data_map.clear(); - _run_length_null_map = &run_length_null_map; - _selection_indices = selected_indices; - _selected_count = selected_count; - _num_values = num_values; - _num_filtered = num_values - selected_count; - _has_filter = selected_count != num_values; - _uses_index_selection = _has_filter; - _read_index = 0; - _num_nulls = 0; - - bool is_null = false; - for (const auto run_length : run_length_null_map) { - if (is_null) { - _num_nulls += run_length; - } - is_null = !is_null; - } - - if (null_map != nullptr && selected_count != 0) { - const size_t null_map_offset = null_map->size(); - null_map->resize(null_map_offset + selected_count); - if (run_length_null_map.empty() || _num_nulls == 0) { - memset(null_map->data() + null_map_offset, 0, selected_count); - } else if (_num_nulls == num_values) { - memset(null_map->data() + null_map_offset, 1, selected_count); - } else { - size_t run_index = 0; - size_t run_end = run_length_null_map[0]; - bool selected_row_is_null = false; - for (size_t i = 0; i < selected_count; ++i) { - const size_t selected_row = selected_indices == nullptr ? i : selected_indices[i]; - while (selected_row >= run_end) { - ++run_index; - selected_row_is_null = !selected_row_is_null; - run_end += run_length_null_map[run_index]; - } - (*null_map)[null_map_offset + i] = selected_row_is_null; - } - } - } - - _reset_selection_cursor(); - return Status::OK(); -} - -void ColumnSelectVector::_reset_selection_cursor() { - _selected_index = 0; - _selection_row_index = 0; - _selection_null_run_index = 0; - _selection_row_is_null = false; - - if (_run_length_null_map->empty()) { - _selection_null_run_remaining = _num_values; - return; - } - - _selection_null_run_remaining = (*_run_length_null_map)[0]; - while (_selection_null_run_remaining == 0 && - _selection_null_run_index + 1 < _run_length_null_map->size()) { - ++_selection_null_run_index; - _selection_row_is_null = !_selection_row_is_null; - _selection_null_run_remaining = (*_run_length_null_map)[_selection_null_run_index]; - } -} - -ColumnSelectVector::DataReadType ColumnSelectVector::_current_selection_type() const { - const bool selected = _selected_index < _selected_count && - _selection_indices[_selected_index] == _selection_row_index; - if (selected) { - return _selection_row_is_null ? NULL_DATA : CONTENT; - } - return _selection_row_is_null ? FILTERED_NULL : FILTERED_CONTENT; -} - -void ColumnSelectVector::_advance_selection_cursor(size_t run_length) { - DCHECK_GT(run_length, 0); - DCHECK_LE(run_length, _selection_null_run_remaining); - const size_t next_row_index = _selection_row_index + run_length; - while (_selected_index < _selected_count && - _selection_indices[_selected_index] < next_row_index) { - ++_selected_index; - } - _selection_row_index = next_row_index; - _selection_null_run_remaining -= run_length; - - while (_selection_row_index < _num_values && _selection_null_run_remaining == 0) { - ++_selection_null_run_index; - _selection_row_is_null = !_selection_row_is_null; - _selection_null_run_remaining = (*_run_length_null_map)[_selection_null_run_index]; - } -} - -size_t ColumnSelectVector::_get_next_selection_run(DataReadType* data_read_type) { - if (_selection_row_index == _num_values) { - return 0; - } - - *data_read_type = _current_selection_type(); - size_t run_length = 0; - if (*data_read_type == CONTENT || *data_read_type == NULL_DATA) { - const size_t null_run_end = _selection_row_index + _selection_null_run_remaining; - size_t selected_index = _selected_index; - size_t expected_row = _selection_row_index; - while (selected_index < _selected_count && - _selection_indices[selected_index] == expected_row && expected_row < null_run_end) { - ++selected_index; - ++expected_row; - } - run_length = selected_index - _selected_index; - } else { - const size_t next_selected_row = _selected_index == _selected_count - ? _num_values - : _selection_indices[_selected_index]; - run_length = - std::min(_selection_null_run_remaining, next_selected_row - _selection_row_index); - } - DCHECK_GT(run_length, 0); - _advance_selection_cursor(run_length); - return run_length; -} - ParsedVersion::ParsedVersion(std::string application, std::optional version, std::optional app_build_hash) : _application(std::move(application)), diff --git a/be/src/format/parquet/parquet_common.h b/be/src/format/parquet/parquet_common.h index 994b2723aefde5..277979932ff2d3 100644 --- a/be/src/format/parquet/parquet_common.h +++ b/be/src/format/parquet/parquet_common.h @@ -93,29 +93,6 @@ class ColumnSelectVector { NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, const std::unordered_set* skipped_indices = nullptr); - /** - * Build a decoder plan from sorted logical-row indices. - * - * This is the native late-materialization contract used by the new Parquet reader. The - * selection indices address the logical rows described by `run_length_null_map`, including - * null rows. They must be strictly increasing and smaller than `num_values`. A null pointer is - * the dense identity selection and therefore requires `selected_count == num_values`; it is - * also accepted for an empty selection. - * - * `run_length_null_map` alternates non-null and null run lengths, starting with a non-null run. - * Zero-length runs are valid because they preserve the alternation. An empty vector means that - * all logical rows are non-null. When `null_map` is non-null, this method appends exactly - * `selected_count` entries in output order. - * - * The plan does not copy `selected_indices` or `run_length_null_map`. Both must remain alive - * and unchanged until the decoder has consumed all runs through get_next_run(). The plan owns - * no per-row action array: it merges the two sorted streams while the decoder advances, so a - * fragmented selection does not allocate O(num_values) scratch memory. - */ - Status init_from_selection(const std::vector& run_length_null_map, size_t num_values, - NullMap* null_map, const uint16_t* selected_indices, - size_t selected_count); - size_t num_values() const { return _num_values; } size_t num_nulls() const { return _num_nulls; } @@ -128,9 +105,6 @@ class ColumnSelectVector { size_t get_next_run(DataReadType* data_read_type) { DCHECK_EQ(_has_filter, has_filter); if constexpr (has_filter) { - if (_uses_index_selection) { - return _get_next_selection_run(data_read_type); - } if (_read_index == _num_values) { return 0; } @@ -147,14 +121,6 @@ class ColumnSelectVector { *data_read_type = type; return run_length; } else { - if (_run_length_null_map->empty()) { - if (_read_index != 0) { - return 0; - } - ++_read_index; - *data_read_type = CONTENT; - return _num_values; - } size_t run_length = 0; while (run_length == 0) { if (_read_index == (*_run_length_null_map).size()) { @@ -168,33 +134,14 @@ class ColumnSelectVector { } private: - Status _validate_null_runs(const std::vector& run_length_null_map, - size_t num_values) const; - void _reset_selection_cursor(); - DataReadType _current_selection_type() const; - void _advance_selection_cursor(size_t run_length); - size_t _get_next_selection_run(DataReadType* data_read_type); - std::vector _data_map; // the length of non-null values and null values are arranged in turn. - const std::vector* _run_length_null_map = nullptr; - bool _has_filter = false; - size_t _num_values = 0; - size_t _num_nulls = 0; - size_t _num_filtered = 0; - size_t _read_index = 0; - - // INDEX selection mode. The input arrays are borrowed for the duration of one decode call. - // `_selection_row_index` is the logical row (nulls included); decoder payload cursors advance - // only for CONTENT and FILTERED_CONTENT runs. - const uint16_t* _selection_indices = nullptr; - bool _uses_index_selection = false; - size_t _selected_count = 0; - size_t _selected_index = 0; - size_t _selection_row_index = 0; - size_t _selection_null_run_index = 0; - size_t _selection_null_run_remaining = 0; - bool _selection_row_is_null = false; + const std::vector* _run_length_null_map; + bool _has_filter; + size_t _num_values; + size_t _num_nulls; + size_t _num_filtered; + size_t _read_index; }; enum class ColumnOrderName { UNDEFINED, TYPE_DEFINED_ORDER }; diff --git a/be/src/format/parquet/vparquet_column_chunk_reader.cpp b/be/src/format/parquet/vparquet_column_chunk_reader.cpp index 722e6613bc5176..b4b919f187073c 100644 --- a/be/src/format/parquet/vparquet_column_chunk_reader.cpp +++ b/be/src/format/parquet/vparquet_column_chunk_reader.cpp @@ -21,9 +21,7 @@ #include #include -#include #include -#include #include #include @@ -51,33 +49,14 @@ struct IOContext; } // namespace doris namespace doris { -ParquetColumnChunkSchema ParquetColumnChunkSchema::from_field_schema( - const FieldSchema& field_schema) { - return ParquetColumnChunkSchema(field_schema.physical_type, - field_schema.parquet_schema.__isset.type_length - ? field_schema.parquet_schema.type_length - : -1, - field_schema.definition_level, field_schema.repetition_level, - field_schema.repeated_parent_def_level); -} - template ColumnChunkReader::ColumnChunkReader( io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx) - : ColumnChunkReader(reader, column_chunk, - ParquetColumnChunkSchema::from_field_schema(*field_schema), - offset_index, total_rows, io_ctx, page_read_ctx) {} - -template -ColumnChunkReader::ColumnChunkReader( - io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, - ParquetColumnChunkSchema chunk_schema, const tparquet::OffsetIndex* offset_index, - size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx) - : _chunk_schema(chunk_schema), - _max_rep_level(chunk_schema.max_repetition_level), - _max_def_level(chunk_schema.max_definition_level), + : _field_schema(field_schema), + _max_rep_level(field_schema->repetition_level), + _max_def_level(field_schema->definition_level), _stream_reader(reader), _metadata(column_chunk->meta_data), _offset_index(offset_index), @@ -109,9 +88,9 @@ Status ColumnChunkReader::skip_nested_values( for (size_t idx = 0; idx < def_levels.size(); idx++) { level_t def_level = def_levels[idx]; - if (IN_COLLECTION && def_level < _chunk_schema.repeated_parent_definition_level) { + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { no_value_cnt++; - } else if (def_level < _chunk_schema.max_definition_level) { + } else if (def_level < _field_schema->definition_level) { no_value_cnt++; } else { value_cnt++; @@ -123,58 +102,6 @@ Status ColumnChunkReader::skip_nested_values( return Status::OK(); } -template -Status ColumnChunkReader::decode_flat_values( - MutableColumnPtr& doris_column, DataTypePtr& data_type, size_t num_values, - const uint16_t* selected_indices, size_t selected_count, NullMap* const null_map, - bool is_dict_filter) { - DCHECK(!IN_COLLECTION); - DCHECK_EQ(_max_rep_level, 0); - - _definition_runs.clear(); - if (_max_def_level != 0) { - // The first run is always non-null. A leading zero preserves parity when the first decoded - // level is null. Two zero runs preserve parity when one logical run exceeds uint16_t. - _definition_runs.emplace_back(0); - bool current_run_is_null = false; - size_t levels_read = 0; - while (levels_read < num_values) { - level_t definition_level = -1; - const size_t run_length = - _def_level_decoder.get_next_run(&definition_level, num_values - levels_read); - if (run_length == 0) { - return Status::Corruption( - "Parquet definition-level stream ended after {} of {} flat values", - levels_read, num_values); - } - - const bool run_is_null = definition_level < _max_def_level; - if (run_is_null != current_run_is_null) { - _definition_runs.emplace_back(0); - current_run_is_null = run_is_null; - } - size_t remaining = run_length; - while (remaining != 0) { - const size_t available = - std::numeric_limits::max() - _definition_runs.back(); - const size_t appended = std::min(remaining, available); - _definition_runs.back() += static_cast(appended); - remaining -= appended; - if (remaining != 0) { - _definition_runs.emplace_back(0); - _definition_runs.emplace_back(0); - } - } - levels_read += run_length; - } - } - - ColumnSelectVector select_vector; - RETURN_IF_ERROR(select_vector.init_from_selection(_definition_runs, num_values, null_map, - selected_indices, selected_count)); - return decode_values(doris_column, data_type, select_vector, is_dict_filter); -} - template Status ColumnChunkReader::_parse_first_page_header() { RETURN_IF_ERROR(parse_page_header()); @@ -784,7 +711,7 @@ Status ColumnChunkReader::load_cross_page_nested_ro template int32_t ColumnChunkReader::_get_type_length() { - switch (_chunk_schema.physical_type) { + switch (_field_schema->physical_type) { case tparquet::Type::INT32: [[fallthrough]]; case tparquet::Type::FLOAT: @@ -796,7 +723,7 @@ int32_t ColumnChunkReader::_get_type_length() { case tparquet::Type::INT96: return 12; case tparquet::Type::FIXED_LEN_BYTE_ARRAY: - return _chunk_schema.type_length; + return _field_schema->parquet_schema.type_length; default: return -1; } diff --git a/be/src/format/parquet/vparquet_column_chunk_reader.h b/be/src/format/parquet/vparquet_column_chunk_reader.h index 31629a6f05f73a..b117f6c6652e7e 100644 --- a/be/src/format/parquet/vparquet_column_chunk_reader.h +++ b/be/src/format/parquet/vparquet_column_chunk_reader.h @@ -70,34 +70,6 @@ struct ColumnChunkReaderStatistics { int64_t page_cache_decompressed_hit_counter = 0; }; -/** - * Immutable physical schema needed by the page/encoding kernel for one leaf column. - * - * Keep this contract independent of both Arrow's ColumnDescriptor and Doris' legacy - * FieldSchema. A native v2 reader can build it directly from the Thrift footer, while the v1 - * reader uses from_field_schema() during migration. Logical-type conversion belongs above this - * layer: the chunk reader only needs the physical carrier, fixed width, and Dremel thresholds to - * decide which payload values exist. - */ -struct ParquetColumnChunkSchema { - ParquetColumnChunkSchema(tparquet::Type::type physical_type, int32_t type_length, - level_t max_definition_level, level_t max_repetition_level, - level_t repeated_parent_definition_level) - : physical_type(physical_type), - type_length(type_length), - max_definition_level(max_definition_level), - max_repetition_level(max_repetition_level), - repeated_parent_definition_level(repeated_parent_definition_level) {} - - const tparquet::Type::type physical_type; - const int32_t type_length; - const level_t max_definition_level; - const level_t max_repetition_level; - const level_t repeated_parent_definition_level; - - static ParquetColumnChunkSchema from_field_schema(const FieldSchema& field_schema); -}; - /** * Read and decode parquet column data into doris block column. *

Usage:

@@ -125,10 +97,6 @@ class ColumnChunkReader { FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_row, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx); - ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, - ParquetColumnChunkSchema chunk_schema, - const tparquet::OffsetIndex* offset_index, size_t total_row, - io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx); ~ColumnChunkReader() = default; // Initialize chunk reader, will generate the decoder and codec. @@ -165,25 +133,6 @@ class ColumnChunkReader { Status decode_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector, bool is_dict_filter); - /** - * Decode selected rows from the current flat data-page slice. - * - * `num_values` is the number of logical rows to consume from the definition-level stream. - * `selected_indices` uses the same sorted, batch-relative contract as - * ColumnSelectVector::init_from_selection(). Selected nulls append defaults to `doris_column` - * and set bits in `null_map`; unselected non-nulls still advance the encoded payload cursor. - * The method therefore advances exactly `num_values` logical rows while materializing exactly - * `selected_count` rows. - * - * This entry point is deliberately limited to non-repeated leaves. LIST/MAP/STRUCT first build - * a shared level plan because their selection is expressed in parent rows rather than leaf - * level positions. Keeping those contracts separate prevents a flat fast path from silently - * misaligning repeated children. - */ - Status decode_flat_values(MutableColumnPtr& doris_column, DataTypePtr& data_type, - size_t num_values, const uint16_t* selected_indices, - size_t selected_count, NullMap* null_map, bool is_dict_filter); - // Get the repetition level decoder of current page. LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } // Get the definition level decoder of current page. @@ -291,7 +240,7 @@ class ColumnChunkReader { } ColumnChunkReaderState _state = NOT_INIT; - const ParquetColumnChunkSchema _chunk_schema; + FieldSchema* _field_schema = nullptr; const level_t _max_rep_level; const level_t _max_def_level; @@ -329,8 +278,6 @@ class ColumnChunkReader { // Map: encoding -> Decoder // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding std::unordered_map> _decoders; - // Alternating non-null/null runs for decode_flat_values(). Capacity is retained across pages. - std::vector _definition_runs; ColumnChunkReaderStatistics _chunk_statistics; }; diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 74d60aa9c62519..22eff697276013 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -111,10 +111,9 @@ instructions as well; this file adds format-v2-specific review expectations. ### Parquet Native Decode Kernel - Keep the production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and - performance baseline, not the migration target. Do not change the v1 reader call path merely to - exercise v2 code. A shared-kernel change under `be/src/format/parquet/` is acceptable only when - extraction cannot reasonably avoid it, the legacy API and behavior remain unchanged, and focused - v1 compatibility tests accompany the v2 tests. + performance baseline, not the migration target. Do not modify `be/src/format/parquet/` for a v2 + decoder change. Reimplement the required decoder under the v2 tree and keep v1 unchanged so + differential correctness and performance results remain meaningful. - Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, and Dremel-level thresholds. Review constructor arguments and stored references for ownership and @@ -164,7 +163,11 @@ instructions as well; this file adds format-v2-specific review expectations. - Materialize directly into Doris columns when the physical and target layouts allow it. Decimal and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. A direct - path may not bypass the conversion rules used by the fallback path. + path may not bypass the conversion rules used by the general conversion path. +- Do not add an Arrow runtime fallback. Once v2 selects its native Parquet reader, unsupported + physical types, encodings, page layouts, or malformed inputs return an explicit status. Arrow may + be used as a test oracle only; no Arrow array, builder, RecordReader, or metadata lifetime belongs + in the completed v2 runtime path. - Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across batches. String-like decoders should gather selected `StringRef` values and append once per batch, rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp index e8aec17605eb92..08ecf5cb81d036 100644 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp @@ -35,6 +35,7 @@ #include "core/data_type_serde/decoded_column_view.h" #include "core/string_ref.h" #include "runtime/runtime_profile.h" +#include "util/defer_op.h" #include "util/simd/bits.h" namespace doris::format::parquet { @@ -354,7 +355,12 @@ Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader // materialization. if (batch->is_binary_value()) { _discarded_binary_chunks.clear(); - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &_discarded_binary_chunks)); + auto status = get_binary_chunks(_name, record_reader, &_discarded_binary_chunks); + // GetBuilderChunks()/GetResult() transfers the Arrow builder result into shared_ptrs. + // Retaining those shared_ptrs in persistent scratch would keep the whole payload alive + // until the next levels-only batch and overlap it with the next builder allocation. + _discarded_binary_chunks.clear(); + RETURN_IF_ERROR(status); } // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the @@ -381,6 +387,14 @@ Status ParquetLeafReader::append_values_with_type(const ParquetLeafBatch& batch, _compact_binary_values.clear(); _spaced_values.clear(); _float_values.clear(); + Defer clear_logical_scratch([this] { + // StringRef entries borrow Arrow buffers owned by ParquetLeafBatch. Drop every borrowed + // pointer before the owner releases those chunks; clear() retains all reusable capacity. + _binary_values.clear(); + _compact_binary_values.clear(); + _spaced_values.clear(); + _float_values.clear(); + }); DecodedColumnView view; view.value_kind = batch._value_kind; view.time_unit = decoded_time_unit(_type_descriptor.time_unit); @@ -511,6 +525,11 @@ Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch _name); } + // A caller normally releases chunks immediately after consuming the batch. Clear once more at + // the producer boundary so an error in an earlier consumer can never overlap the previous + // Arrow payload with Reserve()/ReadRecords() for this batch. + batch->release_binary_chunks(); + try { _record_reader->Reset(); _record_reader->Reserve(batch_rows); @@ -529,7 +548,13 @@ Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, *rows_read); } - return collect_batch(*_record_reader, batch); + auto status = collect_batch(*_record_reader, batch); + if (!status.ok()) { + // collect_batch() may already have acquired Arrow builder chunks before detecting a + // malformed chunk. No consumer will run on an error, so release that ownership here. + batch->release_binary_chunks(); + } + return status; } Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, @@ -558,9 +583,11 @@ Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_sl int16_t value_slot_repetition_level) const { int64_t records_read = 0; RETURN_IF_ERROR(read_batch(batch_rows, &_nested_leaf_batch, &records_read)); - return build_nested_batch_from_leaf_batch(_nested_leaf_batch, records_read, - value_slot_definition_level, batch, - value_slot_repetition_level); + Defer release_binary_chunks([this] { _nested_leaf_batch.release_binary_chunks(); }); + auto status = build_nested_batch_from_leaf_batch(_nested_leaf_batch, records_read, + value_slot_definition_level, batch, + value_slot_repetition_level); + return status; } Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h index 2b2f6121027201..6a5cb34254b465 100644 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h @@ -96,6 +96,11 @@ class ParquetLeafBatch { return _binary_chunks; } + // Release Arrow payload ownership as soon as the synchronous Doris materialization finishes. + // clear() keeps vector capacity, so the batch remains allocation-reusable without extending + // the lifetime of what can be a very large BinaryArray/DictionaryArray payload. + void release_binary_chunks() { _binary_chunks.clear(); } + private: friend class ParquetLeafReader; diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp index d34a8b94b29842..1d8a76f5d0de69 100644 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp @@ -28,6 +28,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type_serde/decoded_column_view.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "util/defer_op.h" #include "util/simd/bits.h" namespace doris::format::parquet { @@ -148,6 +149,10 @@ Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* auto& reader = leaf_reader(); RETURN_IF_ERROR(reader.read_batch(rows, &_leaf_batch, rows_read)); + // Every path below consumes the Arrow payload synchronously. Keep reusable vector capacity but + // release the shared_ptr ownership before this call returns, including validation failures. + Defer release_binary_chunks([this] { _leaf_batch.release_binary_chunks(); }); + _null_map.clear(); RETURN_IF_ERROR(reader.build_null_map(_leaf_batch, *rows_read, &_null_map)); const auto value_kind = decoded_value_kind(_type_descriptor); @@ -323,6 +328,7 @@ Status ScalarColumnReader::read_range_with_dictionary_filter( } RETURN_IF_ERROR(leaf_reader().read_batch(rows, &_leaf_batch, rows_read)); + Defer release_binary_chunks([this] { _leaf_batch.release_binary_chunks(); }); int64_t matched_rows = 0; RETURN_IF_ERROR(append_dictionary_filtered_values(_leaf_batch.binary_chunks(), dictionary_filter, column, row_filter, @@ -353,6 +359,11 @@ Status ScalarColumnReader::append_dictionary_filtered_values( *used_filter = false; _dictionary_binary_values.clear(); + Defer clear_dictionary_binary_values([this] { + // StringRef entries borrow Arrow dictionary bytes. Clear them on every return path before + // read_range_with_dictionary_filter() releases the owning chunks. + _dictionary_binary_values.clear(); + }); for (const auto& chunk : chunks) { DORIS_CHECK(chunk != nullptr); const auto* dict_array = dynamic_cast(chunk.get()); @@ -398,11 +409,7 @@ Status ScalarColumnReader::append_dictionary_filtered_values( if (!*used_filter) { return Status::OK(); } - auto status = append_decoded_binary_values(_dictionary_binary_values, column); - // StringRef does not own the Arrow dictionary bytes. Drop the logical references immediately - // after synchronous materialization while keeping vector capacity for the next batch. - _dictionary_binary_values.clear(); - return status; + return append_decoded_binary_values(_dictionary_binary_values, column); } Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, diff --git a/be/test/format/parquet/byte_array_dict_decoder_test.cpp b/be/test/format/parquet/byte_array_dict_decoder_test.cpp index 5c2f2b75b91657..2e3f398ef56e5a 100644 --- a/be/test/format/parquet/byte_array_dict_decoder_test.cpp +++ b/be/test/format/parquet/byte_array_dict_decoder_test.cpp @@ -178,33 +178,6 @@ TEST_F(ByteArrayDictDecoderTest, test_decode_with_filter_and_null) { } } -TEST_F(ByteArrayDictDecoderTest, test_fragmented_index_selection_with_nulls) { - MutableColumnPtr column = ColumnString::create(); - DataTypePtr data_type = std::make_shared(); - - // Four dictionary zeroes followed by dictionary index 2. Null logical rows consume no index. - std::vector rle_data = {2, 8, 0, 3, 0b00000010, 0}; - Slice data_slice(reinterpret_cast(rle_data.data()), rle_data.size()); - ASSERT_TRUE(_decoder.set_data(&data_slice).ok()); - - const std::vector null_runs = {4, 1, 1, 1}; - const std::vector selection = {1, 4, 5, 6}; - NullMap null_map; - ColumnSelectVector select_vector; - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 7, &null_map, selection.data(), - selection.size()) - .ok()); - - ASSERT_TRUE(_decoder.decode_values(column, data_type, select_vector, false).ok()); - ASSERT_EQ(column->size(), 4); - EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1})); - EXPECT_EQ(column->get_data_at(0).to_string(), "apple"); - EXPECT_EQ(column->get_data_at(1).to_string(), ""); - EXPECT_EQ(column->get_data_at(2).to_string(), "cherry"); - EXPECT_EQ(column->get_data_at(3).to_string(), ""); -} - // Test empty dictionary case TEST_F(ByteArrayDictDecoderTest, test_empty_dict) { ByteArrayDictDecoder empty_decoder; diff --git a/be/test/format/parquet/byte_array_plain_decoder_test.cpp b/be/test/format/parquet/byte_array_plain_decoder_test.cpp index cba01b83511578..ad78492a345ccd 100644 --- a/be/test/format/parquet/byte_array_plain_decoder_test.cpp +++ b/be/test/format/parquet/byte_array_plain_decoder_test.cpp @@ -244,45 +244,6 @@ TEST_F(ByteArrayPlainDecoderTest, test_skip_value) { EXPECT_EQ(result_column->get_data_at(0).to_string(), "cherry"); } -TEST_F(ByteArrayPlainDecoderTest, test_fragmented_index_selection_with_nulls) { - const char* values[] = {"zero", "two", "three", "five"}; - size_t data_size = 0; - for (const auto* value : values) { - data_size += sizeof(uint32_t) + strlen(value); - } - _data = std::make_unique(data_size); - size_t offset = 0; - for (const auto* value : values) { - const auto length = static_cast(strlen(value)); - encode_fixed32_le(_data.get() + offset, length); - offset += sizeof(uint32_t); - memcpy(_data.get() + offset, value, length); - offset += length; - } - _data_slice = Slice(_data.get(), data_size); - - ByteArrayPlainDecoder decoder; - ASSERT_TRUE(decoder.set_data(&_data_slice).ok()); - MutableColumnPtr column = ColumnString::create(); - DataTypePtr data_type = std::make_shared(); - const std::vector null_runs = {1, 1, 2, 1, 1}; - const std::vector selection = {1, 2, 4, 5}; - NullMap null_map; - ColumnSelectVector select_vector; - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 6, &null_map, selection.data(), - selection.size()) - .ok()); - - ASSERT_TRUE(decoder.decode_values(column, data_type, select_vector, false).ok()); - ASSERT_EQ(column->size(), 4); - EXPECT_EQ(null_map, (NullMap {1, 0, 1, 0})); - EXPECT_EQ(column->get_data_at(0).to_string(), ""); - EXPECT_EQ(column->get_data_at(1).to_string(), "two"); - EXPECT_EQ(column->get_data_at(2).to_string(), ""); - EXPECT_EQ(column->get_data_at(3).to_string(), "five"); -} - TEST_F(ByteArrayPlainDecoderTest, test_decode_truncated_length_prefix) { uint8_t data[] = {0x01, 0x00}; _data_slice = Slice(data, sizeof(data)); diff --git a/be/test/format/parquet/byte_stream_split_decoder_test.cpp b/be/test/format/parquet/byte_stream_split_decoder_test.cpp index d298cf5205dd48..b332a13449a99d 100644 --- a/be/test/format/parquet/byte_stream_split_decoder_test.cpp +++ b/be/test/format/parquet/byte_stream_split_decoder_test.cpp @@ -168,38 +168,6 @@ TEST_F(ByteStreamSplitDecoderTest, test_basic_decode_fixed_length_object) { EXPECT_EQ(fixed_length_value(*result_column, 2), "ghi"); } -TEST_F(ByteStreamSplitDecoderTest, test_fragmented_index_selection_with_nulls) { - const std::vector values = {1.0F, 2.0F, 3.0F, 4.0F}; - std::vector encoded(values.size() * sizeof(float)); - for (size_t value_index = 0; value_index < values.size(); ++value_index) { - const auto* bytes = reinterpret_cast(&values[value_index]); - for (size_t byte_index = 0; byte_index < sizeof(float); ++byte_index) { - encoded[byte_index * values.size() + value_index] = bytes[byte_index]; - } - } - Slice data_slice(encoded.data(), encoded.size()); - ASSERT_TRUE(_decoder.set_data(&data_slice).ok()); - _decoder.set_type_length(sizeof(float)); - - MutableColumnPtr column = ColumnFloat32::create(); - DataTypePtr data_type = std::make_shared(); - const std::vector null_runs = {2, 1, 2, 1}; - const std::vector selection = {1, 2, 4, 5}; - NullMap null_map; - ColumnSelectVector select_vector; - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 6, &null_map, selection.data(), - selection.size()) - .ok()); - - ASSERT_TRUE(_decoder.decode_values(column, data_type, select_vector, false).ok()); - ASSERT_EQ(column->size(), 4); - EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1})); - const auto& decoded = assert_cast(*column).get_data(); - EXPECT_FLOAT_EQ(decoded[0], 2.0F); - EXPECT_FLOAT_EQ(decoded[2], 4.0F); -} - // Test decoding with filter for FLOAT type TEST_F(ByteStreamSplitDecoderTest, test_decode_with_filter_float) { // Prepare test data for FLOAT type diff --git a/be/test/format/parquet/delta_bit_pack_decoder_test.cpp b/be/test/format/parquet/delta_bit_pack_decoder_test.cpp index 8a0a2bd1940e64..78a857b3442468 100644 --- a/be/test/format/parquet/delta_bit_pack_decoder_test.cpp +++ b/be/test/format/parquet/delta_bit_pack_decoder_test.cpp @@ -157,32 +157,6 @@ TEST_F(DeltaBitPackDecoderTest, test_decode_with_filter_and_null) { } } -TEST_F(DeltaBitPackDecoderTest, test_fragmented_index_selection_with_nulls) { - std::vector encoded_data = { - // Four values 10, 11, 12, 13 with a constant delta of one. - 0x80, 0x01, 0x04, 0x04, 0x14, 0x02, 0x00, 0x00, 0x00, 0x00}; - Slice data_slice(reinterpret_cast(encoded_data.data()), encoded_data.size()); - ASSERT_TRUE(_decoder->set_data(&data_slice).ok()); - - MutableColumnPtr column = ColumnInt32::create(); - DataTypePtr data_type = std::make_shared(); - const std::vector null_runs = {2, 1, 2, 1}; - const std::vector selection = {1, 2, 4, 5}; - NullMap null_map; - ColumnSelectVector select_vector; - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 6, &null_map, selection.data(), - selection.size()) - .ok()); - - ASSERT_TRUE(_decoder->decode_values(column, data_type, select_vector, false).ok()); - ASSERT_EQ(column->size(), 4); - EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1})); - const auto& decoded = assert_cast(*column).get_data(); - EXPECT_EQ(decoded[0], 11); - EXPECT_EQ(decoded[2], 13); -} - // Test skipping values for delta bit pack decoding TEST_F(DeltaBitPackDecoderTest, test_skip_value) { // Prepare encoded data diff --git a/be/test/format/parquet/fix_length_plain_decoder_test.cpp b/be/test/format/parquet/fix_length_plain_decoder_test.cpp index e431385a138b9e..78b992c2a36416 100644 --- a/be/test/format/parquet/fix_length_plain_decoder_test.cpp +++ b/be/test/format/parquet/fix_length_plain_decoder_test.cpp @@ -240,35 +240,6 @@ TEST_F(FixLengthPlainDecoderTest, test_decode_with_filter_and_null) { } } -TEST_F(FixLengthPlainDecoderTest, test_fragmented_index_selection_with_nulls) { - int32_t values[] = {123, 456, 789, 1011}; - _data = std::make_unique(sizeof(values)); - memcpy(_data.get(), values, sizeof(values)); - _data_slice = Slice(_data.get(), sizeof(values)); - - FixLengthPlainDecoder decoder; - decoder.set_type_length(sizeof(int32_t)); - ASSERT_TRUE(decoder.set_data(&_data_slice).ok()); - - MutableColumnPtr column = ColumnInt32::create(); - DataTypePtr data_type = std::make_shared(); - const std::vector null_runs = {1, 1, 2, 1, 1}; - const std::vector selection = {1, 2, 4, 5}; - NullMap null_map; - ColumnSelectVector select_vector; - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 6, &null_map, selection.data(), - selection.size()) - .ok()); - - ASSERT_TRUE(decoder.decode_values(column, data_type, select_vector, false).ok()); - ASSERT_EQ(column->size(), 4); - EXPECT_EQ(null_map, (NullMap {1, 0, 1, 0})); - const auto& decoded = assert_cast(*column).get_data(); - EXPECT_EQ(decoded[1], 456); - EXPECT_EQ(decoded[3], 1011); -} - // Test skipping values TEST_F(FixLengthPlainDecoderTest, test_skip_value) { // Prepare test data: create fixed-length integer values diff --git a/be/test/format/parquet/parquet_common_test.cpp b/be/test/format/parquet/parquet_common_test.cpp index 7ac8f168f5bc54..2bd0dcc253a16c 100644 --- a/be/test/format/parquet/parquet_common_test.cpp +++ b/be/test/format/parquet/parquet_common_test.cpp @@ -454,220 +454,4 @@ TEST_F(ColumnSelectVectorTest, test_filter_map_index) { EXPECT_EQ(type, ColumnSelectVector::CONTENT); } -TEST_F(ColumnSelectVectorTest, test_index_selection_merges_selection_and_null_runs) { - const std::vector null_runs = {2, 2, 3, 1}; - const std::vector selection = {0, 2, 3, 6}; - NullMap null_map = {9}; - ColumnSelectVector select_vector; - - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 8, &null_map, selection.data(), - selection.size()) - .ok()); - EXPECT_TRUE(select_vector.has_filter()); - EXPECT_EQ(select_vector.num_values(), 8); - EXPECT_EQ(select_vector.num_nulls(), 3); - EXPECT_EQ(select_vector.num_filtered(), 4); - EXPECT_EQ(null_map, (NullMap {9, 0, 1, 1, 0})); - - using ReadType = ColumnSelectVector::DataReadType; - const std::vector> expected_runs = { - {1, ReadType::CONTENT}, {1, ReadType::FILTERED_CONTENT}, - {2, ReadType::NULL_DATA}, {2, ReadType::FILTERED_CONTENT}, - {1, ReadType::CONTENT}, {1, ReadType::FILTERED_NULL}, - }; - for (const auto& [expected_length, expected_type] : expected_runs) { - ReadType type; - EXPECT_EQ(select_vector.get_next_run(&type), expected_length); - EXPECT_EQ(type, expected_type); - } - ReadType type; - EXPECT_EQ(select_vector.get_next_run(&type), 0); -} - -TEST_F(ColumnSelectVectorTest, test_dense_index_selection_uses_null_runs_directly) { - const std::vector null_runs = {2, 0, 1, 2, 1}; - NullMap null_map; - ColumnSelectVector select_vector; - - ASSERT_TRUE(select_vector.init_from_selection(null_runs, 6, &null_map, nullptr, 6).ok()); - EXPECT_FALSE(select_vector.has_filter()); - EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 1, 0})); - - using ReadType = ColumnSelectVector::DataReadType; - ReadType type; - EXPECT_EQ(select_vector.get_next_run(&type), 2); - EXPECT_EQ(type, ReadType::CONTENT); - EXPECT_EQ(select_vector.get_next_run(&type), 1); - EXPECT_EQ(type, ReadType::CONTENT); - EXPECT_EQ(select_vector.get_next_run(&type), 2); - EXPECT_EQ(type, ReadType::NULL_DATA); - EXPECT_EQ(select_vector.get_next_run(&type), 1); - EXPECT_EQ(type, ReadType::CONTENT); - EXPECT_EQ(select_vector.get_next_run(&type), 0); -} - -TEST_F(ColumnSelectVectorTest, test_empty_index_selection_skips_content_and_nulls) { - const std::vector null_runs = {2, 2, 3, 1}; - NullMap null_map = {7}; - ColumnSelectVector select_vector; - - ASSERT_TRUE(select_vector.init_from_selection(null_runs, 8, &null_map, nullptr, 0).ok()); - EXPECT_TRUE(select_vector.has_filter()); - EXPECT_EQ(select_vector.num_filtered(), 8); - EXPECT_EQ(null_map, (NullMap {7})); - - using ReadType = ColumnSelectVector::DataReadType; - ReadType type; - EXPECT_EQ(select_vector.get_next_run(&type), 2); - EXPECT_EQ(type, ReadType::FILTERED_CONTENT); - EXPECT_EQ(select_vector.get_next_run(&type), 2); - EXPECT_EQ(type, ReadType::FILTERED_NULL); - EXPECT_EQ(select_vector.get_next_run(&type), 3); - EXPECT_EQ(type, ReadType::FILTERED_CONTENT); - EXPECT_EQ(select_vector.get_next_run(&type), 1); - EXPECT_EQ(type, ReadType::FILTERED_NULL); - EXPECT_EQ(select_vector.get_next_run(&type), 0); -} - -TEST_F(ColumnSelectVectorTest, test_empty_null_runs_mean_all_non_null) { - const std::vector null_runs; - const std::vector selection = {1, 4}; - NullMap null_map; - ColumnSelectVector select_vector; - - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, 6, &null_map, selection.data(), - selection.size()) - .ok()); - EXPECT_EQ(null_map, (NullMap {0, 0})); - - using ReadType = ColumnSelectVector::DataReadType; - const std::vector> expected_runs = { - {1, ReadType::FILTERED_CONTENT}, {1, ReadType::CONTENT}, - {2, ReadType::FILTERED_CONTENT}, {1, ReadType::CONTENT}, - {1, ReadType::FILTERED_CONTENT}, - }; - for (const auto& [expected_length, expected_type] : expected_runs) { - ReadType type; - EXPECT_EQ(select_vector.get_next_run(&type), expected_length); - EXPECT_EQ(type, expected_type); - } -} - -TEST_F(ColumnSelectVectorTest, test_index_selection_rejects_invalid_contracts) { - ColumnSelectVector select_vector; - const std::vector null_runs = {4}; - const std::vector duplicate = {1, 1}; - const std::vector descending = {2, 1}; - const std::vector outside = {4}; - - EXPECT_FALSE(select_vector.init_from_selection(null_runs, 4, nullptr, nullptr, 2).ok()); - EXPECT_FALSE( - select_vector - .init_from_selection(null_runs, 4, nullptr, duplicate.data(), duplicate.size()) - .ok()); - EXPECT_FALSE(select_vector - .init_from_selection(null_runs, 4, nullptr, descending.data(), - descending.size()) - .ok()); - EXPECT_FALSE( - select_vector.init_from_selection(null_runs, 4, nullptr, outside.data(), outside.size()) - .ok()); - EXPECT_FALSE(select_vector.init_from_selection(null_runs, 4, nullptr, nullptr, 5).ok()); - EXPECT_FALSE( - select_vector.init_from_selection(std::vector {3}, 4, nullptr, nullptr, 4) - .ok()); - EXPECT_FALSE( - select_vector.init_from_selection(std::vector {5}, 4, nullptr, nullptr, 4) - .ok()); -} - -TEST_F(ColumnSelectVectorTest, test_reinitialization_resets_selection_mode) { - const std::vector null_runs = {4}; - const std::vector selection = {1}; - ColumnSelectVector select_vector; - ASSERT_TRUE( - select_vector - .init_from_selection(null_runs, 4, nullptr, selection.data(), selection.size()) - .ok()); - - std::vector filter_data = {1, 0, 1, 0}; - FilterMap filter_map; - ASSERT_TRUE(filter_map.init(filter_data.data(), filter_data.size(), false).ok()); - ASSERT_TRUE(select_vector.init(null_runs, 4, nullptr, &filter_map, 0).ok()); - - using ReadType = ColumnSelectVector::DataReadType; - ReadType type; - EXPECT_EQ(select_vector.get_next_run(&type), 1); - EXPECT_EQ(type, ReadType::CONTENT); - EXPECT_EQ(select_vector.get_next_run(&type), 1); - EXPECT_EQ(type, ReadType::FILTERED_CONTENT); -} - -TEST_F(ColumnSelectVectorTest, test_index_selection_exhaustive_small_batches) { - using ReadType = ColumnSelectVector::DataReadType; - for (size_t num_values = 1; num_values <= 6; ++num_values) { - const size_t bitmap_count = size_t {1} << num_values; - for (size_t null_bitmap = 0; null_bitmap < bitmap_count; ++null_bitmap) { - std::vector null_runs; - bool current_run_is_null = false; - uint16_t current_run_length = 0; - for (size_t row = 0; row < num_values; ++row) { - const bool row_is_null = ((null_bitmap >> row) & 1) != 0; - if (row_is_null == current_run_is_null) { - ++current_run_length; - continue; - } - null_runs.push_back(current_run_length); - current_run_length = 1; - current_run_is_null = row_is_null; - } - null_runs.push_back(current_run_length); - - for (size_t selection_bitmap = 0; selection_bitmap < bitmap_count; ++selection_bitmap) { - std::vector selection; - NullMap expected_null_map; - std::vector expected_actions; - for (size_t row = 0; row < num_values; ++row) { - const bool row_is_null = ((null_bitmap >> row) & 1) != 0; - const bool row_is_selected = ((selection_bitmap >> row) & 1) != 0; - if (row_is_selected) { - selection.push_back(static_cast(row)); - expected_null_map.push_back(row_is_null); - } - expected_actions.push_back( - row_is_selected - ? (row_is_null ? ReadType::NULL_DATA : ReadType::CONTENT) - : (row_is_null ? ReadType::FILTERED_NULL - : ReadType::FILTERED_CONTENT)); - } - - NullMap null_map; - ColumnSelectVector select_vector; - ASSERT_TRUE(select_vector - .init_from_selection(null_runs, num_values, &null_map, - selection.data(), selection.size()) - .ok()); - EXPECT_EQ(null_map, expected_null_map); - - std::vector actual_actions; - ReadType type; - if (selection.size() == num_values) { - while (const size_t run_length = select_vector.get_next_run(&type)) { - actual_actions.insert(actual_actions.end(), run_length, type); - } - } else { - while (const size_t run_length = select_vector.get_next_run(&type)) { - actual_actions.insert(actual_actions.end(), run_length, type); - } - } - EXPECT_EQ(actual_actions, expected_actions) - << "num_values=" << num_values << ", null_bitmap=" << null_bitmap - << ", selection_bitmap=" << selection_bitmap; - } - } - } -} - } // namespace doris diff --git a/be/test/format/parquet/parquet_thrift_test.cpp b/be/test/format/parquet/parquet_thrift_test.cpp index d67170dca7743c..0fe101db598138 100644 --- a/be/test/format/parquet/parquet_thrift_test.cpp +++ b/be/test/format/parquet/parquet_thrift_test.cpp @@ -71,22 +71,6 @@ class ParquetThriftReaderTest : public testing::Test { void TearDown() override { TimezoneUtils::clear_timezone_caches(); } }; -TEST_F(ParquetThriftReaderTest, column_chunk_schema_is_independent_of_field_schema_lifetime) { - FieldSchema field; - field.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; - field.definition_level = 5; - field.repetition_level = 2; - field.repeated_parent_def_level = 4; - field.parquet_schema.__set_type_length(16); - - const auto chunk_schema = ParquetColumnChunkSchema::from_field_schema(field); - EXPECT_EQ(chunk_schema.physical_type, tparquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(chunk_schema.type_length, 16); - EXPECT_EQ(chunk_schema.max_definition_level, 5); - EXPECT_EQ(chunk_schema.max_repetition_level, 2); - EXPECT_EQ(chunk_schema.repeated_parent_definition_level, 4); -} - TEST_F(ParquetThriftReaderTest, normal) { auto local_fs = io::global_local_filesystem(); io::FileReaderSPtr reader; diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp index 0d0f9a2f8567cc..438aac91847bbd 100644 --- a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp @@ -148,6 +148,14 @@ struct ParquetLeafReaderTestAccess { value_slot_definition_level, nested_batch, value_slot_repetition_level); } + + static size_t binary_value_size(const ParquetLeafReader& reader) { + return reader._binary_values.size(); + } + + static size_t binary_value_capacity(const ParquetLeafReader& reader) { + return reader._binary_values.capacity(); + } }; std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, @@ -275,6 +283,9 @@ TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { auto status = reader.append_values(batch, 5, &null_map, column); ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(ParquetLeafReaderTestAccess::binary_value_size(reader), 0); + EXPECT_GE(ParquetLeafReaderTestAccess::binary_value_capacity(reader), 5); + const auto& nullable = assert_cast(*column); const auto& strings = assert_cast(nullable.get_nested_column()); ASSERT_EQ(nullable.size(), 5); @@ -285,6 +296,23 @@ TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); } +TEST(ParquetLeafReaderTest, ReleaseBinaryChunksDropsPayloadAndRetainsVectorCapacity) { + ParquetLeafBatch batch; + batch._binary_chunks.reserve(4); + auto array = fixed_binary_array({"payload"}, 7); + std::weak_ptr payload = array; + batch._binary_chunks.push_back(array); + array.reset(); + + const auto capacity = batch.binary_chunks().capacity(); + ASSERT_FALSE(payload.expired()); + batch.release_binary_chunks(); + + EXPECT_TRUE(payload.expired()); + EXPECT_TRUE(batch.binary_chunks().empty()); + EXPECT_EQ(batch.binary_chunks().capacity(), capacity); +} + TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { ParquetTypeDescriptor descriptor; descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; @@ -308,6 +336,7 @@ TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { EXPECT_FALSE(status.ok()); EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), std::string::npos); + EXPECT_EQ(ParquetLeafReaderTestAccess::binary_value_size(reader), 0); } TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 5de3efbe3e7b7c..b7a2b34fdbe024 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -504,6 +504,7 @@ TEST(ParquetScalarColumnReaderTest, DictionaryIndexOutsideFilterIsCorruption) { reader, chunks, IColumn::Filter {1}, column, &row_filter, &matched_rows, &used_filter); EXPECT_EQ(ErrorCode::CORRUPTION, status.code()) << status; EXPECT_NE(status.to_string().find("Invalid parquet dictionary index 1"), std::string::npos); + EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_size(reader), 0); } TEST(ParquetScalarColumnReaderTest, LeafReaderAndDictionaryScratchArePersistent) { diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index c88ede9974dc95..6661406f1934b6 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -73,22 +73,23 @@ mature Parquet page and encoding support: | Stage | State and boundary | | --- | --- | -| Native encoding kernel | Reuse and extract the Doris v1 decoders behind a schema-independent physical Column Chunk contract. Selection-aware scalar decoding and reusable string scratch are the first vertical slice. | +| Native encoding kernel | Reimplement the proven Doris v1 decoder algorithms under `be/src/format_v2/parquet/` behind a schema-independent physical Column Chunk contract. Selection-aware scalar decoding and reusable string scratch are the first vertical slice. | | Native column reader | Make the leaf reader and its SerDe, null map, selection ranges, binary values, level buffers, and conversion scratch persistent for a Row Group. Add direct Doris materialization paths. | | Complex reconstruction | Build one shared Dremel level plan per requested parent-row range and use it for STRUCT/ARRAY/MAP siblings, offsets, and null maps. | | Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | -| Compatibility removal | Remove Arrow data-read adapters only after type/encoding/page/writer compatibility and performance gates pass. Keep an explicit fallback during migration; never silently select a partially supported native path. | +| Compatibility removal | Remove Arrow data-read adapters after type/encoding/page/writer compatibility and performance gates pass. Production v2 never falls back from a selected native reader to Arrow; an unsupported combination returns an explicit error. | -At the current migration boundary, Arrow may still be present in the v2 footer/schema, planning, or -I/O adapter path. That does not make Arrow arrays part of the native decoder contract. Documentation -and Profile names must distinguish the native data-page path from remaining metadata/adaptation -work so an intermediate patch is not mistaken for complete Arrow removal. +At the current migration boundary, Arrow is still present in the v2 footer/schema, planning, and +data path. Such a patch is transitional and must not be described as a completed native decoder. +The production target contains no Arrow object in the v2 runtime call chain. Arrow may be used only +as a test oracle while differential tests are being developed, never as a runtime compatibility +fallback. All production integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the -correctness and performance control. Extracting a narrow shared decoder primitive is permitted when -it preserves the legacy API and behavior, but the migration does not route v1 reads through a new -v2 path. This separation makes differential testing meaningful and prevents a v2 performance -experiment from regressing the established reader. +correctness and performance control. Decoder code required by v2 is reimplemented in the v2 tree; +the migration does not change v1 or route v1 reads through a new v2 path. This separation makes +differential testing meaningful and prevents a v2 experiment from regressing the established +reader. ### 2.2 Native Interface Ownership @@ -383,9 +384,9 @@ that parsing step both feed the same level and value-decoder contracts. | DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value and binary scratch state | | BYTE_STREAM_SPLIT | Reassemble primitive lanes and apply selection without an Arrow intermediate | -Unsupported physical-type/encoding combinations return an explicit error or use a declared -compatibility fallback. They never produce a plausible result through a decoder selected only by -logical Doris type. Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated +Unsupported physical-type/encoding combinations return an explicit error. They never fall back to +Arrow and never produce a plausible result through a decoder selected only by logical Doris type. +Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. ### 7.3 Direct Materialization and Scratch Reuse @@ -396,17 +397,22 @@ runs. String-like decoders gather selected `StringRef` values into persistent sc one batched append, so fragmented predicates do not cause one destination growth or copy call per run. Scratch stores references only while the backing page/dictionary buffer is stable. +`DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded +physical values and is useful to generic format conversion code, but routing every Parquet value +through it would recreate an intermediate materialization layer. A native encoding decoder consumes +encoded page bytes plus levels/selection and appends to the Doris physical column. Only a genuine +physical-to-logical mismatch invokes the reusable conversion layer after decode. + Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian two's-complement values with correct sign extension, and apply precision/scale conversion exactly once. Date, timestamp, INT96, unsigned annotations, CHAR/VARCHAR, and timezone conversions retain the same semantic checks as the general conversion path. A fast path is enabled only when those checks prove the result is equivalent. -The persistent leaf reader owns reusable SerDe/conversion objects, null map, selection ranges, +The persistent leaf reader owns reusable conversion objects, null map, selection ranges, definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris -column/builder capacity. Logical sizes are reset at batch boundaries while capacity is retained. -During migration, any remaining Arrow builder fallback follows the same lifetime and capacity-reuse -rule; it must not be constructed once per `ReadRecords` call. +column capacity. Logical sizes are reset at batch boundaries while capacity is retained. The native +path does not create an Arrow builder or Arrow array. ### 7.4 Complex Types and Shared Level Plans @@ -472,7 +478,8 @@ leaf as the entry-shape owner and validates the value leaf against the same entr one representative leaf for parent validity and only checks sibling alignment while decoding each child. -During migration, the four Arrow-facing methods remain a compatibility facade. Their leaf reader, +During migration, the four Arrow-facing methods are temporary scaffolding, not a runtime fallback. +Their leaf reader, SerDe, binary/null/level scratch, selection ranges, nested batches, parent nulls, entry counts, and child-column handles must be persistent so the facade does not add per-batch allocation churn. New native decoder code must not depend on this phase ordering or reproduce the temporary From ebcd1bcb911f3025874c53351f3dcac36ca48d33 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 13:32:55 +0800 Subject: [PATCH 03/34] [improvement](be) Decode Parquet v2 columns natively Issue Number: None Related PR: #65674 Problem Summary: FileScannerV2 materialized ordinary Parquet values through Arrow arrays and temporary nested batches, which added allocation, conversion, and peak-memory overhead compared with the v1 reader. Route ordinary predicate and output columns through a persistent native reader that decodes directly into Doris columns, preserves selection and page-index coordinates, shares v1-compatible footer/page caches and MergeRange I/O, and keeps Arrow only for metadata planning, dictionary probing, and the existing COUNT complex levels-only path. Add detailed native decode profiles, adaptive-batch fragmentation profiles, complex/string/decimal/fixed-binary coverage, and update the Parquet design and review guidance. No FE or COUNT pushdown behavior is changed, and no be/src/format decoder code is modified. Improve FileScannerV2 Parquet scan memory use and native decode performance for scalar and complex columns. - Test: Unit Test - 192 format v2 Parquet BE unit tests on the designated remote host - Behavior changed: Yes (ordinary FileScannerV2 Parquet values use native direct materialization; query results and pushdown semantics are unchanged) - Does this need documentation: Yes (design and review guide updated in this change) --- be/src/format_v2/AGENTS.md | 24 +- .../parquet/parquet_file_context.cpp | 152 ++++- .../format_v2/parquet/parquet_file_context.h | 52 +- be/src/format_v2/parquet/parquet_profile.cpp | 38 ++ be/src/format_v2/parquet/parquet_profile.h | 36 + be/src/format_v2/parquet/parquet_reader.cpp | 11 +- be/src/format_v2/parquet/parquet_scan.cpp | 117 ++-- be/src/format_v2/parquet/parquet_scan.h | 16 +- .../parquet/reader/native_column_reader.cpp | 632 ++++++++++++++++++ .../parquet/reader/native_column_reader.h | 127 ++++ .../format_v2/parquet/parquet_reader_test.cpp | 250 ++++++- docs/file-scanner-v2-parquet-scan-design.md | 57 +- 12 files changed, 1409 insertions(+), 103 deletions(-) create mode 100644 be/src/format_v2/parquet/reader/native_column_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native_column_reader.h diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 22eff697276013..5b6007550fb94b 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -110,10 +110,11 @@ instructions as well; this file adds format-v2-specific review expectations. ### Parquet Native Decode Kernel -- Keep the production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and - performance baseline, not the migration target. Do not modify `be/src/format/parquet/` for a v2 - decoder change. Reimplement the required decoder under the v2 tree and keep v1 unchanged so - differential correctness and performance results remain meaningful. +- Keep new production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and + performance baseline. A v2 adapter may reuse the unchanged native kernel while its semantics are + identical; do not modify `be/src/format/parquet/` for a v2 decoder change. When v2 requires a + decoder change, reimplement that decoder under the v2 tree and keep v1 unchanged so differential + correctness and performance results remain meaningful. - Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, and Dremel-level thresholds. Review constructor arguments and stored references for ownership and @@ -136,6 +137,10 @@ instructions as well; this file adds format-v2-specific review expectations. row count, selection, and `VALUES` versus `LEVELS_ONLY` mode. Shape parsing, payload-cursor advancement, validation, and output must have an obvious owner; callers must not depend on an undocumented load-before-build phase. +- `ParquetLeafBatch` belongs only to that isolated Arrow facade and its levels-only migration path. + Ordinary predicate/output scans must construct `NativeColumnReader` and may not route decoded + values through `ParquetLeafBatch`, `DecodedColumnView`, Arrow arrays, or temporary nested Doris + columns before the final destination column. - Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one level traversal and share the result. For example, `[[1, 2], NULL, []]` must yield entry counts `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning the same levels per child. MAP key @@ -164,10 +169,11 @@ instructions as well; this file adds format-v2-specific review expectations. and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. A direct path may not bypass the conversion rules used by the general conversion path. -- Do not add an Arrow runtime fallback. Once v2 selects its native Parquet reader, unsupported +- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, unsupported physical types, encodings, page layouts, or malformed inputs return an explicit status. Arrow may - be used as a test oracle only; no Arrow array, builder, RecordReader, or metadata lifetime belongs - in the completed v2 runtime path. + be used by the explicitly documented metadata-planning or levels-only migration paths and as a + test oracle; no Arrow array, builder, RecordReader, or metadata lifetime belongs in ordinary + value materialization. - Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across batches. String-like decoders should gather selected `StringRef` values and append once per batch, rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded @@ -185,6 +191,10 @@ instructions as well; this file adds format-v2-specific review expectations. invalidation, admission, and fallback I/O. Any intentional difference needs benchmark and memory evidence showing it is no worse for v1 workloads. Cache lookup must never alter page ordinal, decoder, level, or dictionary cursor state. +- Apply v1's MergeRange decision to the native data-page reader, after metadata/dictionary probes + finish. Predicate and lazy readers for one Row Group must share one ordered-range wrapper, and + native per-column prefetch must be disabled while that wrapper is active. Never allocate one + MergeRange buffer per leaf; wide complex projections would multiply its bounded scratch memory. - Preserve observability inside aggregate counters. `TotalBatches` must be decomposable into probe, dense, selected, empty, page-crossing, and nested/fragmented work where relevant; decode, level, selection, conversion, allocation, and materialization time must remain attributable without diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index aa8f2622ade43b..27cc579ecb2975 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -29,16 +30,21 @@ #include #include +#include "common/cast_set.h" #include "common/check.h" #include "common/config.h" +#include "format/parquet/parquet_thrift_util.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" +#include "io/fs/file_meta_cache.h" #include "io/fs/file_reader.h" #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" +#include "runtime/exec_env.h" #include "storage/cache/page_cache.h" #include "util/slice.h" +#include "util/thrift_util.h" namespace doris::format::parquet { @@ -373,10 +379,10 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { static_cast(range.end_offset())); } - // This mirrors the v1 parquet reader: when projected column chunks in a row group are - // small random IOs, make the actual ReadAt path range-aware. Arrow still drives decoding, - // but every page read below this point sees MergeRangeFileReader instead of the raw remote - // reader, so adjacent small requests can be coalesced and served from merge buffers. + // This mirrors the v1 parquet reader for the migration metadata/index ReadAt path. Native + // data-page decoding owns a separate BufferedFileStreamReader and v1-compatible page cache; + // adjacent metadata/index requests here can still be coalesced and served from merge + // buffers. // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once // and satisfy the following ReadAt calls from its boxes, reducing remote request count. @@ -569,17 +575,60 @@ Status arrow_status_to_doris_status(const arrow::Status& status) { } Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, - bool enable_page_cache, - const io::FileDescription& file_description) { + bool enable_page_cache, const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz) { DORIS_CHECK(input_file_reader != nullptr); + native_file = input_file_reader; + native_io_ctx = io_ctx; + + // Use the exact footer cache key and payload type used by v1. This deliberately happens before + // Arrow metadata is opened: native readers can reuse a footer produced by a v1 scan (and vice + // versa), and a cache miss performs one bounded tail read through the same Doris FileReader. + auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); + const auto meta_cache_key = FileMetaCache::get_key(native_file, file_description); + size_t native_footer_size = 0; + if (meta_cache != nullptr && meta_cache->enabled() && + meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { + native_metadata = native_meta_cache_handle.data(); + ++native_footer_cache_hits; + } else { + RETURN_IF_ERROR(parse_thrift_footer( + native_file, &native_metadata_owner, &native_footer_size, io_ctx, + /*enable_mapping_varbinary=*/true, enable_mapping_timestamp_tz)); + ++native_footer_read_calls; + if (meta_cache != nullptr && meta_cache->enabled()) { + meta_cache->insert(meta_cache_key, native_metadata_owner.release(), + &native_meta_cache_handle); + native_metadata = native_meta_cache_handle.data(); + } else { + native_metadata = native_metadata_owner.get(); + } + } + DORIS_CHECK(native_metadata != nullptr); + const_cast(native_metadata->schema()).assign_ids(); + auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); - arrow_file = std::make_shared(std::move(input_file_reader), io_ctx, - enable_page_cache, - std::move(page_cache_file_key)); + native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); + arrow_file = std::make_shared( + input_file_reader, io_ctx, enable_page_cache, std::move(page_cache_file_key)); try { - // TODO: Cache parquet metadata in file system layer to avoid repeated metadata read for same file. + // Arrow metadata is still used by the v2 pruning planner during the migration, but it must + // not trigger a second footer read. Re-serialize the immutable cached Thrift object and + // hand the parsed Arrow metadata into Open(). The serialized buffer is needed only during + // FileMetaData::Make(), while the v1-compatible cache remains the single footer owner. + ThriftSerializer serializer(/*compact=*/true, + static_cast(std::max(native_footer_size, 4096))); + std::vector serialized_metadata; + RETURN_IF_ERROR(serializer.serialize( + const_cast(&native_metadata->to_thrift()), + &serialized_metadata)); + uint32_t serialized_size = cast_set(serialized_metadata.size()); + auto arrow_metadata = + ::parquet::FileMetaData::Make(serialized_metadata.data(), &serialized_size, + ::parquet::default_reader_properties()); + DORIS_CHECK(static_cast(serialized_size) == serialized_metadata.size()); this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties()); + arrow_file, ::parquet::default_reader_properties(), std::move(arrow_metadata)); metadata = this->file_reader->metadata(); schema = metadata != nullptr ? metadata->schema() : nullptr; } catch (const ::parquet::ParquetException& e) { @@ -602,6 +651,49 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont return Status::OK(); } +Status ParquetFileContext::load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const { + DORIS_CHECK(offset_indexes != nullptr); + offset_indexes->clear(); + if (leaf_column_ids.empty() || file_reader == nullptr) { + return Status::OK(); + } + try { + auto page_index_reader = file_reader->GetPageIndexReader(); + if (page_index_reader == nullptr) { + return Status::OK(); + } + auto row_group_reader = page_index_reader->RowGroup(row_group_id); + if (row_group_reader == nullptr) { + return Status::OK(); + } + for (const int leaf_column_id : leaf_column_ids) { + auto arrow_index = row_group_reader->GetOffsetIndex(leaf_column_id); + if (arrow_index == nullptr) { + continue; + } + tparquet::OffsetIndex native_index; + native_index.page_locations.reserve(arrow_index->page_locations().size()); + for (const auto& arrow_location : arrow_index->page_locations()) { + tparquet::PageLocation native_location; + native_location.__set_offset(arrow_location.offset); + native_location.__set_compressed_page_size(arrow_location.compressed_page_size); + native_location.__set_first_row_index(arrow_location.first_row_index); + native_index.page_locations.push_back(std::move(native_location)); + } + offset_indexes->emplace(leaf_column_id, std::move(native_index)); + } + } catch (const ::parquet::ParquetException&) { + // OffsetIndex is optional. Selected logical ranges still enforce correctness, while the + // native reader conservatively falls back to sequential page traversal. + offset_indexes->clear(); + } catch (const std::exception&) { + offset_indexes->clear(); + } + return Status::OK(); +} + void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { DORIS_CHECK(arrow_file != nullptr); static_cast(arrow_file.get()) @@ -622,9 +714,37 @@ bool ParquetFileContext::set_random_access_ranges(const std::vectorset_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); } +bool ParquetFileContext::set_native_random_access_ranges( + const std::vector& ranges, size_t avg_io_size, + RuntimeProfile* profile, int64_t merge_read_slice_size) { + DORIS_CHECK(native_file != nullptr); + if (!detail::should_use_merge_range_reader( + ranges, avg_io_size, + typeid_cast(native_file.get()) != nullptr)) { + native_row_group_file = native_file; + return false; + } + + const auto valid_ranges = detail::valid_prefetch_ranges(ranges); + std::vector native_ranges; + native_ranges.reserve(valid_ranges.size()); + for (const auto& range : valid_ranges) { + native_ranges.emplace_back(cast_set(range.offset), + cast_set(range.end_offset())); + } + std::ranges::sort(native_ranges, {}, &io::PrefetchRange::start_offset); + native_row_group_file = std::make_shared( + profile, native_file, native_ranges, merge_read_slice_size); + return true; +} + void ParquetFileContext::reset_random_access_ranges() { DORIS_CHECK(arrow_file != nullptr); static_cast(arrow_file.get())->reset_random_access_ranges(); + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); } ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { @@ -635,6 +755,10 @@ ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { } Status ParquetFileContext::close() { + if (native_row_group_file != nullptr && native_row_group_file != native_file) { + native_row_group_file->collect_profile_before_close(); + } + native_row_group_file.reset(); if (file_reader != nullptr) { try { file_reader->Close(); @@ -646,6 +770,12 @@ Status ParquetFileContext::close() { } file_reader.reset(); arrow_file.reset(); + native_metadata = nullptr; + native_metadata_owner.reset(); + native_meta_cache_handle = {}; + native_file.reset(); + native_io_ctx = nullptr; + native_page_cache_enabled = false; return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 67e9102139dbf2..72eacd9a9adeb0 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -24,10 +24,13 @@ #include #include #include +#include #include #include "common/status.h" +#include "format/parquet/vparquet_file_metadata.h" #include "io/fs/file_reader.h" +#include "util/obj_lru_cache.h" namespace doris::io { struct FileDescription; @@ -142,32 +145,59 @@ bool should_use_merge_range_reader(const std::vector& ran } // namespace detail struct ParquetFileContext { + // The native data-page path reads from Doris' FileReader directly. Keep this handle separate + // from the Arrow RandomAccessFile used by the metadata/index migration path so opening Arrow + // metadata never transfers ownership away from the native reader. + io::FileReaderSPtr native_file; + // Row-group-scoped view of native_file. Small projected chunks use the same + // MergeRangeFileReader policy as v1; large chunks and in-memory files keep native_file. + io::FileReaderSPtr native_row_group_file; + io::IOContext* native_io_ctx = nullptr; + // V1-compatible Thrift footer/schema used to construct Doris' native page/encoding readers. + // A cache hit is owned by native_meta_cache_handle; a miss without cache is owned by + // native_metadata_owner. + const FileMetaData* native_metadata = nullptr; + std::unique_ptr native_metadata_owner; + ObjLRUCache::CacheHandle native_meta_cache_handle; + int64_t native_footer_read_calls = 0; + int64_t native_footer_cache_hits = 0; + bool native_page_cache_enabled = false; + std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, - const io::FileDescription& file_description); - // Register file ranges that belong to selected Parquet column chunks. Arrow still owns page - // decoding, so v2 caches the serialized bytes read inside these ranges and excludes - // footer/metadata reads that happen before registration. + const io::FileDescription& file_description, + bool enable_mapping_timestamp_tz = false); + Status load_native_offset_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* offset_indexes) const; + // Register ranges for the remaining Arrow metadata/index adapter. Native data pages use the + // v1-compatible page cache owned by BufferedFileStreamReader instead. void register_page_cache_ranges(std::vector ranges); // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Switch the active reader used by Arrow ReadAt() to v1's MergeRangeFileReader when the current - // row group's projected column chunks are small random IOs. This is the real v1-compatible - // prefetch path: subsequent Arrow page reads go through the merged reader instead of merely - // warming file cache in the background. Returns true when merge-range reading is active. + // Switch the active reader used by Arrow metadata/index ReadAt() to MergeRangeFileReader when + // projected chunks are small random IOs. Native page decoding is intentionally independent. bool set_random_access_ranges(const std::vector& ranges, size_t avg_io_size, RuntimeProfile* profile, int64_t merge_read_slice_size); - // Restore Arrow ReadAt() to the base Doris file reader and flush any active merge-reader - // counters. Row-group setup uses this before dictionary-page probes, because those probes are - // a separate pass over the column chunk from the later Arrow RecordReader data-page stream. + // Install the v1-compatible MergeRangeFileReader on the native data-page path. Dictionary + // probes must run before this method because their Arrow ReadAt order is independent of the + // sequential projected chunk ranges consumed by MergeRangeFileReader. + bool set_native_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size); + const io::FileReaderSPtr& native_data_file() const { + return native_row_group_file != nullptr ? native_row_group_file : native_file; + } + // Restore both Arrow and native ReadAt() to the base Doris file reader and flush active + // merge-reader counters. Row-group setup uses this before dictionary-page probes. void reset_random_access_ranges(); ParquetPageCacheStats page_cache_stats() const; Status close(); diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 29783e14a05071..9f37ea2ca49882 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -59,6 +59,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); + dense_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DenseBatches", TUnit::UNIT, parquet_profile, 1); + selected_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedBatches", TUnit::UNIT, + parquet_profile, 1); empty_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptySelectionBatches", TUnit::UNIT, parquet_profile, 1); range_gap_skipped_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RangeGapSkippedRows", @@ -75,6 +79,14 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowSkipRecordsTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, + parquet_profile, 1); + native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", + TUnit::UNIT, parquet_profile, 1); + page_crossing_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageCrossingBatches", + TUnit::UNIT, parquet_profile, 1); + nested_batches = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1); filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, @@ -202,6 +214,30 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .arrow_read_records_time = arrow_read_records_time, .arrow_skip_records_time = arrow_skip_records_time, .materialization_time = materialization_time, + .decompress_time = decompress_time, + .decompress_count = decompress_cnt, + .decode_header_time = decode_header_time, + .decode_value_time = decode_value_time, + .decode_dictionary_time = decode_dict_time, + .decode_level_time = decode_level_time, + .decode_null_map_time = decode_null_map_time, + .convert_time = convert_time, + .page_index_read_calls = page_index_read_calls, + .skip_page_header_count = skip_page_header_num, + .parse_page_header_count = parse_page_header_num, + .read_page_header_time = read_page_header_time, + .page_read_count = page_read_counter, + .page_cache_write_count = page_cache_write_counter, + .page_cache_compressed_write_count = page_cache_compressed_write_counter, + .page_cache_decompressed_write_count = page_cache_decompressed_write_counter, + .page_cache_hit_count = page_cache_hit_counter, + .page_cache_miss_count = page_cache_missing_counter, + .page_cache_compressed_hit_count = page_cache_compressed_hit_counter, + .page_cache_decompressed_hit_count = page_cache_decompressed_hit_counter, + .native_read_calls = native_read_calls, + .native_page_fragments = native_page_fragments, + .page_crossing_batches = page_crossing_batches, + .nested_batches = nested_batches, }; } @@ -212,6 +248,8 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .rows_filtered_by_conjunct = rows_filtered_by_conjunct, .lazy_read_filtered_rows = lazy_read_filtered_rows, .total_batches = total_batches, + .dense_batches = dense_batches, + .selected_batches = selected_batches, .empty_selection_batches = empty_selection_batches, .range_gap_skipped_rows = range_gap_skipped_rows, .column_read_time = column_read_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 827cac45d94df0..e1b8abee3dff5c 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -37,6 +37,33 @@ struct ParquetColumnReaderProfile { RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) RuntimeProfile::Counter* arrow_skip_records_time = nullptr; // Arrow SkipRecords time (ns) RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + // Native page/encoding reader internals. These counters intentionally mirror v1 so a v1/v2 + // profile comparison attributes page IO, decompression, levels, value decode and conversion to + // the same stages. + RuntimeProfile::Counter* decompress_time = nullptr; + RuntimeProfile::Counter* decompress_count = nullptr; + RuntimeProfile::Counter* decode_header_time = nullptr; + RuntimeProfile::Counter* decode_value_time = nullptr; + RuntimeProfile::Counter* decode_dictionary_time = nullptr; + RuntimeProfile::Counter* decode_level_time = nullptr; + RuntimeProfile::Counter* decode_null_map_time = nullptr; + RuntimeProfile::Counter* convert_time = nullptr; + RuntimeProfile::Counter* page_index_read_calls = nullptr; + RuntimeProfile::Counter* skip_page_header_count = nullptr; + RuntimeProfile::Counter* parse_page_header_count = nullptr; + RuntimeProfile::Counter* read_page_header_time = nullptr; + RuntimeProfile::Counter* page_read_count = nullptr; + RuntimeProfile::Counter* page_cache_write_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_write_count = nullptr; + RuntimeProfile::Counter* page_cache_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_miss_count = nullptr; + RuntimeProfile::Counter* page_cache_compressed_hit_count = nullptr; + RuntimeProfile::Counter* page_cache_decompressed_hit_count = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; // v1-native reader calls + RuntimeProfile::Counter* native_page_fragments = nullptr; // page-bounded read fragments + RuntimeProfile::Counter* page_crossing_batches = nullptr; // batches spanning multiple pages + RuntimeProfile::Counter* nested_batches = nullptr; // complex-column read batches }; // ============================================================================ @@ -48,6 +75,9 @@ struct ParquetScanProfile { RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; // rows avoided by late materialization RuntimeProfile::Counter* total_batches = nullptr; // total batch count + RuntimeProfile::Counter* dense_batches = nullptr; // batches retaining every physical row + RuntimeProfile::Counter* selected_batches = + nullptr; // non-empty batches compacted by predicates RuntimeProfile::Counter* empty_selection_batches = nullptr; // empty batches after full filtering RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps @@ -96,6 +126,8 @@ struct ParquetProfile { RuntimeProfile::Counter* selected_rows = nullptr; RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; RuntimeProfile::Counter* total_batches = nullptr; + RuntimeProfile::Counter* dense_batches = nullptr; + RuntimeProfile::Counter* selected_batches = nullptr; RuntimeProfile::Counter* empty_selection_batches = nullptr; RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; @@ -106,6 +138,10 @@ struct ParquetProfile { RuntimeProfile::Counter* arrow_read_records_time = nullptr; RuntimeProfile::Counter* arrow_skip_records_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; + RuntimeProfile::Counter* native_read_calls = nullptr; + RuntimeProfile::Counter* native_page_fragments = nullptr; + RuntimeProfile::Counter* page_crossing_batches = nullptr; + RuntimeProfile::Counter* nested_batches = nullptr; RuntimeProfile::Counter* lazy_read_filtered_rows = nullptr; RuntimeProfile::Counter* filtered_bytes = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index cd84ab74ef953e..03fe9ef50d00f0 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -414,6 +414,7 @@ Status ParquetReader::init(RuntimeState* state) { _state->enable_strict_mode = state->enable_strict_mode(); _state->scheduler.set_timezone(&state->timezone_obj()); _state->scheduler.set_enable_strict_mode(_state->enable_strict_mode); + _state->scheduler.set_runtime_state(state); } int64_t merge_read_slice_size = -1; if (state != nullptr && state->query_options().__isset.merge_read_slice_size) { @@ -423,7 +424,14 @@ Status ParquetReader::init(RuntimeState* state) { _state->scheduler.set_batch_size(_batch_size); // Open parquet file and parse metadata to get file schema. RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description)); + _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz)); + if (_profile != nullptr) { + COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, + _state->file_context.native_footer_read_calls); + COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, + _state->file_context.native_footer_cache_hits); + } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier RETURN_IF_ERROR( @@ -808,6 +816,7 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r Status ParquetReader::close() { if (_state != nullptr) { + _state->scheduler.close(); _sync_page_cache_profile(); RETURN_IF_ERROR(_state->file_context.close()); } diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index eafb741a53161d..5d09a189404bec 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -35,6 +35,7 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/native_column_reader.h" namespace doris::format::parquet { @@ -673,7 +674,7 @@ void ParquetScanScheduler::reset() { } void ParquetScanScheduler::reset_current_row_group() { - _current_row_group.reset(); + _has_current_row_group = false; _current_predicate_columns.clear(); _current_non_predicate_columns.clear(); _current_dictionary_filters.clear(); @@ -683,6 +684,7 @@ void ParquetScanScheduler::reset_current_row_group() { _current_row_group_rows_read = 0; _current_row_group_first_row = 0; _current_selected_ranges.clear(); + _current_offset_indexes.clear(); _current_range_idx = 0; _current_range_rows_read = 0; // Readers are row-group scoped. If every remaining row was filtered, no future output can @@ -702,25 +704,19 @@ Status ParquetScanScheduler::open_next_row_group( const format::FileScanRequest& request, bool* has_row_group) { *has_row_group = false; if (_next_row_group_plan_idx >= _row_group_plans.size()) { + // The last row group's native readers have already been released by + // reset_current_row_group(). Flush the shared merge reader now so its counters are visible + // when EOF is returned and its bounded scratch does not survive until file close. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; return Status::OK(); } const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; const int row_group_idx = row_group_plan.row_group_id; - // Row-level dictionary filters read dictionary pages before Arrow RecordReaders are created. - // Keep that probe on the base reader: MergeRangeFileReader expects each registered range to be - // consumed as one forward pass, while the later RecordReader opens the same column chunk again - // for the data-page stream. + // Row-level dictionary filters still use the migration metadata/page probe. Native data-page + // readers below do not construct an Arrow RowGroupReader or RecordReader. file_context.reset_random_access_ranges(); _current_merge_range_active = false; - try { - _current_row_group = file_context.file_reader->RowGroup(row_group_idx); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to open parquet row group {}: {}", row_group_idx, - e.what()); - } auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); DORIS_CHECK(row_group_metadata != nullptr); @@ -728,10 +724,46 @@ Status ParquetScanScheduler::open_next_row_group( DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); DORIS_CHECK(_current_row_group_rows > 0); _current_row_group_id = row_group_idx; + _has_current_row_group = true; DORIS_CHECK(!row_group_plan.selected_ranges.empty()); _current_row_group_first_row = row_group_plan.first_file_row; _current_row_group_rows_read = 0; _current_selected_ranges = row_group_plan.selected_ranges; + const bool has_filtered_ranges = _current_selected_ranges.size() != 1 || + _current_selected_ranges[0].start != 0 || + _current_selected_ranges[0].length != _current_row_group_rows; + if (has_filtered_ranges) { + std::unordered_set leaf_column_ids; + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + continue; + } + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + DORIS_CHECK(file_schema[local_id] != nullptr); + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); + } + RETURN_IF_ERROR(file_context.load_native_offset_indexes(row_group_idx, leaf_column_ids, + &_current_offset_indexes)); + } + for (const auto& [leaf_column_id, skip_plan] : row_group_plan.page_skip_plans) { + if (!_current_offset_indexes.contains(leaf_column_id)) { + continue; + } + for (size_t page = 0; page < skip_plan.skipped_pages.size(); ++page) { + if (!skip_plan.should_skip_page(page)) { + continue; + } + if (_page_skip_profile.skipped_pages != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); + } + if (_page_skip_profile.skipped_bytes != nullptr) { + COUNTER_UPDATE(_page_skip_profile.skipped_bytes, + skip_plan.skipped_page_compressed_size(page)); + } + } + } _current_range_idx = 0; _current_range_rows_read = 0; _current_predicate_columns.clear(); @@ -739,11 +771,18 @@ Status ParquetScanScheduler::open_next_row_group( _current_dictionary_filters.clear(); RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, row_group_idx, *row_group_metadata)); - _current_merge_range_active = - prepare_current_row_group_reader(file_context, file_schema, request, row_group_idx); + // Dictionary probing is complete, so the native data-page readers can now share the same + // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a + // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy + // materialization for wide schemas. + const auto native_ranges = build_row_group_prefetch_ranges( + *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); + _current_merge_range_active = file_context.set_native_random_access_ranges( + native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, + _merge_read_slice_size); ParquetColumnReaderFactory column_reader_factory( - _current_row_group, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, + nullptr, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, _page_skip_profile, _timezone, _enable_strict_mode, _scan_profile.column_reader_profile); for (const auto& col : request.predicate_columns) { @@ -766,14 +805,17 @@ Status ParquetScanScheduler::open_next_row_group( const auto& column_schema = file_schema[local_id]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR( - column_reader_factory.create(*column_schema, &col, &column_reader, - _current_dictionary_filters.contains(local_id))); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + _current_dictionary_filters.contains(local_id), _scan_profile.column_reader_profile, + &column_reader)); _current_predicate_columns[local_id] = std::move(column_reader); } - // Start warming filter-column chunks as soon as their row group is selected. Parquet v2 still - // reads through Arrow's random-access reader; this prefetch only warms Doris file cache blocks - // in the background and never changes the row/column materialization order. + // Start warming filter-column chunks as soon as their row group is selected. The native + // BufferedFileStreamReader later consumes the same Doris file-cache blocks; prefetch never + // changes row/column materialization order. if (!_current_merge_range_active) { prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, &_current_predicate_prefetched); @@ -800,7 +842,11 @@ Status ParquetScanScheduler::open_next_row_group( const auto& column_schema = file_schema[local_id]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; - RETURN_IF_ERROR(column_reader_factory.create(*column_schema, &col, &column_reader)); + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + false, _scan_profile.column_reader_profile, &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } if (!_current_merge_range_active && request.conjuncts.empty() && @@ -1360,20 +1406,6 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return execute_scheduled_delete_conjuncts(); } -bool ParquetScanScheduler::prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx) { - if (file_context.metadata == nullptr) { - return false; - } - const auto ranges = build_row_group_prefetch_ranges( - *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); - const size_t avg_io_size = detail::average_prefetch_range_size(ranges); - return file_context.set_random_access_ranges(ranges, avg_io_size, _profile, - _merge_read_slice_size); -} - void ParquetScanScheduler::prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, @@ -1437,6 +1469,11 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0 && _scan_profile.empty_selection_batches != nullptr) { COUNTER_UPDATE(_scan_profile.empty_selection_batches, 1); + } else if (static_cast(selected_rows) == batch_rows && + _scan_profile.dense_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.dense_batches, 1); + } else if (_scan_profile.selected_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.selected_batches, 1); } if (need_filter_output && !predicate_columns_filtered) { IColumn::Filter output_filter = selection_to_filter(selection, selected_rows, batch_rows); @@ -1451,7 +1488,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } if (selected_rows == 0) { // Predicate readers have consumed this physical batch, but touching every lazy column here - // turns a long rejected prefix into `empty_batches * lazy_columns` Arrow calls. Record only + // turns a long rejected prefix into `empty_batches * lazy_columns` native calls. Record only // the positional lag. If [0, 32), [32, 64), and [64, 96) are empty, the first surviving // batch performs one skip(96) per lazy column. If the row group ends instead, reset drops the // lazy readers without flushing because no value from them can be observed. @@ -1537,7 +1574,7 @@ Status ParquetScanScheduler::read_next_batch( const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { *rows = 0; while (true) { - if (_current_row_group == nullptr) { + if (!_has_current_row_group) { bool has_row_group = false; RETURN_IF_ERROR( open_next_row_group(file_context, file_schema, request, &has_row_group)); diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index c656b146cefeaa..da0a47aed3f443 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include @@ -128,6 +131,10 @@ class ParquetScanScheduler { void set_enable_strict_mode(bool enable_strict_mode) { _enable_strict_mode = enable_strict_mode; } + void set_runtime_state(RuntimeState* runtime_state) { _runtime_state = runtime_state; } + // Release row-group readers before the owning RuntimeProfile is reported. Native readers + // publish their accumulated page/decode statistics from their destructor. + void close() { reset_current_row_group(); } // Upper scanner owns adaptive memory feedback; scheduler only applies the current row cap when // splitting selected row ranges into physical read batches. void set_batch_size(size_t batch_size) { @@ -170,11 +177,6 @@ class ParquetScanScheduler { const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched); - bool prepare_current_row_group_reader( - ParquetFileContext& file_context, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx); - Status read_current_row_group_batch( ParquetFileContext& file_context, const std::vector>& file_schema, @@ -187,7 +189,7 @@ class ParquetScanScheduler { std::vector _row_group_plans; // row group queue to scan size_t _next_row_group_plan_idx = 0; // index of the next row group to process - std::shared_ptr<::parquet::RowGroupReader> _current_row_group; // Arrow RowGroup reader + bool _has_current_row_group = false; std::map> _current_predicate_columns; // predicate ColumnReaders std::map> @@ -202,6 +204,7 @@ class ParquetScanScheduler { int64_t _current_row_group_first_row = 0; // first file row of the current row group std::vector _current_selected_ranges; // selected ranges for the current row group after page-index pruning + std::unordered_map _current_offset_indexes; size_t _current_range_idx = 0; // current selected_range index int64_t _current_range_rows_read = 0; // rows read in the current range // Predicate readers move immediately because they decide which rows survive. Non-predicate @@ -219,6 +222,7 @@ class ParquetScanScheduler { std::optional _global_rowid_context; const cctz::time_zone* _timezone = nullptr; bool _enable_strict_mode = false; + RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_filtered_rows = 0; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp new file mode 100644 index 00000000000000..92fc5681fa58ec --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -0,0 +1,632 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native_column_reader.h" + +#include +#include +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "core/assert_cast.h" +#include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" +#include "format/parquet/vparquet_file_metadata.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "runtime/runtime_state.h" + +namespace doris::format::parquet { +namespace { + +DataTypePtr projected_type(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection) { + if (!format::is_partial_projection(projection)) { + return schema.type; + } + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + return schema.type; + case ParquetColumnSchemaKind::STRUCT: { + DataTypes child_types; + Strings child_names; + child_types.reserve(projection->children.size()); + child_names.reserve(projection->children.size()); + for (const auto& child_projection : projection->children) { + const auto child_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != schema.children.end()); + child_types.push_back(make_nullable(projected_type(**child_it, &child_projection))); + child_names.push_back((*child_it)->name); + } + DataTypePtr type = std::make_shared(child_types, child_names); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto* child_projection = + format::find_child_projection(projection, schema.children[0]->local_id); + DORIS_CHECK(child_projection != nullptr); + DataTypePtr type = std::make_shared( + projected_type(*schema.children[0], child_projection)); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + case ParquetColumnSchemaKind::MAP: { + DORIS_CHECK(schema.children.size() == 2); + const auto* value_projection = + format::find_child_projection(projection, schema.children[1]->local_id); + DORIS_CHECK(value_projection != nullptr); + DataTypePtr type = std::make_shared( + make_nullable(schema.children[0]->type), + make_nullable(projected_type(*schema.children[1], value_projection))); + return schema.type->is_nullable() ? make_nullable(type) : type; + } + } + DORIS_CHECK(false); + return nullptr; +} + +const FieldSchema* find_child_field(const FieldSchema& parent, const ParquetColumnSchema& child) { + auto field_it = std::ranges::find_if(parent.children, [&](const FieldSchema& field) { + return (child.parquet_field_id >= 0 && field.field_id == child.parquet_field_id) || + field.name == child.name; + }); + return field_it == parent.children.end() ? nullptr : &*field_it; +} + +void collect_projected_ids(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const FieldSchema& native_field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + if (!format::is_partial_projection(projection)) { + return; + } + for (const auto& child_projection : projection->children) { + const auto schema_it = std::ranges::find_if(schema.children, [&](const auto& child) { + return child->local_id == child_projection.local_id(); + }); + DORIS_CHECK(schema_it != schema.children.end()); + const FieldSchema* child_field = find_child_field(native_field, **schema_it); + DORIS_CHECK(child_field != nullptr); + ids->insert(child_field->get_column_id()); + collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + } + if (schema.kind == ParquetColumnSchemaKind::MAP) { + DORIS_CHECK(!native_field.children.empty()); + // MAP entry existence and offsets are owned by the key stream even for value-only + // projections. Keep the key reader live and validate key/value entry alignment in v1's + // native MapColumnReader. + ids->insert(native_field.children[0].get_column_id()); + } +} + +Status append_non_null_dictionary_values(MutableColumnPtr& target, MutableColumnPtr values) { + DORIS_CHECK(target); + DORIS_CHECK(values); + const size_t value_count = values->size(); + if (auto* nullable = check_and_get_column(*target); nullable != nullptr) { + nullable->get_nested_column().insert_range_from(*values, 0, value_count); + auto& null_map = nullable->get_null_map_data(); + null_map.resize_fill(null_map.size() + value_count, 0); + return Status::OK(); + } + target->insert_range_from(*values, 0, value_count); + return Status::OK(); +} + +} // namespace + +NativeColumnReader::NativeColumnReader(const ParquetColumnSchema& schema, + DataTypePtr projected_type, + ParquetColumnReaderProfile profile) + : ParquetColumnReader(schema, std::move(projected_type), profile), + _nested(schema.kind != ParquetColumnSchemaKind::PRIMITIVE) {} + +NativeColumnReader::~NativeColumnReader() { + (void)sync_native_profile(); +} + +Status NativeColumnReader::create( + const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, + io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + if (reader == nullptr) { + return Status::InvalidArgument("Native parquet reader result is null"); + } + if (file == nullptr || metadata == nullptr) { + return Status::InvalidArgument("Native parquet file context is not initialized"); + } + if (row_group_id < 0 || + row_group_id >= static_cast(metadata->to_thrift().row_groups.size())) { + return Status::InvalidArgument("Invalid native parquet row group {}", row_group_id); + } + const auto& native_schema = metadata->schema(); + if (column_schema.local_id < 0 || column_schema.local_id >= native_schema.size()) { + return Status::InvalidArgument("Invalid native parquet top-level column id {} for {}", + column_schema.local_id, column_schema.name); + } + auto* field = const_cast(native_schema.get_column(column_schema.local_id)); + DORIS_CHECK(field != nullptr); + if (field->name != column_schema.name && + !(field->field_id >= 0 && field->field_id == column_schema.parquet_field_id)) { + return Status::Corruption( + "Native/metadata parquet schema mismatch at column {}: native={}, arrow={}", + column_schema.local_id, field->name, column_schema.name); + } + + auto type = projected_type(column_schema, projection); + std::shared_ptr schema_node; + RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(type, *field, + schema_node)); + std::set projected_ids; + collect_projected_ids(column_schema, projection, *field, &projected_ids); + + auto native_reader = std::unique_ptr( + new NativeColumnReader(column_schema, std::move(type), profile)); + RETURN_IF_ERROR(native_reader->init( + std::move(file), metadata, row_group_id, field, std::move(schema_node), + std::move(projected_ids), selected_ranges, offset_indexes, timezone, io_ctx, + runtime_state, enable_page_cache, enable_dictionary_filter)); + *reader = std::move(native_reader); + return Status::OK(); +} + +Status NativeColumnReader::init( + io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, FieldSchema* field, + std::shared_ptr schema_node, + std::set projected_column_ids, const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, bool enable_dictionary_filter) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(field != nullptr); + DORIS_CHECK(schema_node != nullptr); + const auto& row_group = metadata->to_thrift().row_groups[row_group_id]; + DORIS_CHECK(row_group.num_rows > 0); + _row_group_rows = row_group.num_rows; + _selected_ranges = selected_ranges; + DORIS_CHECK(!_selected_ranges.empty()); + for (const auto& range : _selected_ranges) { + DORIS_CHECK(range.start >= 0); + DORIS_CHECK(range.length > 0); + DORIS_CHECK(range.start + range.length <= _row_group_rows); + _row_ranges.add(::doris::RowRange(range.start, range.start + range.length)); + } + _offset_indexes = offset_indexes; + _schema_node = std::move(schema_node); + _projected_column_ids = std::move(projected_column_ids); + _dictionary_filter_enabled = enable_dictionary_filter; + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + const size_t max_buffer_size = std::min(max_group_buffer, max_column_buffer); + RuntimeState* native_runtime_state = runtime_state; + const bool runtime_page_cache_enabled = + runtime_state == nullptr || + runtime_state->query_options().enable_parquet_file_page_cache; + if (runtime_page_cache_enabled != enable_page_cache) { + TQueryOptions query_options = + runtime_state == nullptr ? TQueryOptions() : runtime_state->query_options(); + query_options.__set_enable_parquet_file_page_cache(enable_page_cache); + _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); + native_runtime_state = _page_cache_runtime_state.get(); + } + RETURN_IF_ERROR(::doris::ParquetColumnReader::create( + std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, + max_buffer_size, _offset_indexes, native_runtime_state, false, _projected_column_ids, + _filter_column_ids)); + DORIS_CHECK(_native_reader != nullptr); + _skip_column = _type->create_column(); + return Status::OK(); +} + +Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, MutableColumnPtr& column, + const DataTypePtr& output_type, bool dictionary_ids, + int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(column); + DORIS_CHECK(output_type != nullptr); + DORIS_CHECK(rows_read != nullptr); + *rows_read = 0; + if (rows == 0) { + return Status::OK(); + } + + ::doris::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + ColumnPtr native_column(std::move(column)); + bool eof = false; + int64_t native_calls = 0; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + ++native_calls; + size_t loop_rows = 0; + RETURN_IF_ERROR(_native_reader->read_column_data( + native_column, output_type, _schema_node, filter, + static_cast(rows - *rows_read), &loop_rows, &eof, dictionary_ids)); + if (loop_rows == 0 && !eof) { + // A selected RowRanges plan may reject the current data page completely. V1 advances + // the page cursor and deliberately returns zero rows so the caller can request the + // next page. Bound consecutive empty transitions by the Row Group row count to retain + // a deterministic corruption exit if a decoder ever stops advancing. + if (++consecutive_empty_calls > _row_group_rows + 1) { + column = IColumn::mutate(std::move(native_column)); + return Status::Corruption("Native parquet reader made no progress for column {}", + _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + column = IColumn::mutate(std::move(native_column)); + if (_profile.native_read_calls != nullptr) { + COUNTER_UPDATE(_profile.native_read_calls, native_calls); + } + if (_nested && _profile.nested_batches != nullptr) { + COUNTER_UPDATE(_profile.nested_batches, 1); + } + if (*rows_read != rows) { + return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", + *rows_read, rows, _name); + } + return Status::OK(); +} + +Status NativeColumnReader::validate_selected_span(int64_t rows) { + DORIS_CHECK(rows >= 0); + while (_selected_range_idx < _selected_ranges.size()) { + const auto& range = _selected_ranges[_selected_range_idx]; + const int64_t range_end = range.start + range.length; + if (_logical_row_position < range_end) { + break; + } + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + return Status::Corruption("Native parquet read past selected ranges for column {}", _name); + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start || + rows > range.start + range.length - _logical_row_position) { + return Status::Corruption( + "Native parquet read [{}, {}) crosses selected range [{}, {}) for column {}", + _logical_row_position, _logical_row_position + rows, range.start, + range.start + range.length, _name); + } + return Status::OK(); +} + +void NativeColumnReader::advance_selected_span(int64_t rows) { + _logical_row_position += rows; + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } +} + +Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { + RETURN_IF_ERROR(validate_selected_span(rows)); + RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); + advance_selected_span(*rows_read); + update_reader_read_rows(*rows_read); + record_page_fragments(sync_native_profile()); + return Status::OK(); +} + +Status NativeColumnReader::skip(int64_t rows) { + if (rows <= 0) { + return Status::OK(); + } + DORIS_CHECK(_logical_row_position <= _row_group_rows - rows); + int64_t remaining = rows; + int64_t native_skipped_rows = 0; + while (remaining > 0) { + while (_selected_range_idx < _selected_ranges.size() && + _logical_row_position >= _selected_ranges[_selected_range_idx].start + + _selected_ranges[_selected_range_idx].length) { + ++_selected_range_idx; + } + if (_selected_range_idx >= _selected_ranges.size()) { + _logical_row_position += remaining; + break; + } + const auto& range = _selected_ranges[_selected_range_idx]; + if (_logical_row_position < range.start) { + const int64_t gap = std::min(remaining, range.start - _logical_row_position); + _logical_row_position += gap; + remaining -= gap; + continue; + } + const int64_t selected_rows = + std::min(remaining, range.start + range.length - _logical_row_position); + _skip_column->clear(); + _filter_scratch.resize(static_cast(selected_rows)); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, + _type, false, &rows_read)); + DORIS_CHECK(_skip_column->empty()); + DORIS_CHECK(rows_read == selected_rows); + _logical_row_position += rows_read; + native_skipped_rows += rows_read; + remaining -= rows_read; + } + update_reader_skip_rows(native_skipped_rows); + record_page_fragments(sync_native_profile()); + return Status::OK(); +} + +Status NativeColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column) { + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + _filter_scratch.assign(static_cast(batch_rows), 0); + for (uint16_t idx = 0; idx < selected_rows; ++idx) { + _filter_scratch[selection.get_index(idx)] = 1; + } + const size_t old_size = column->size(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, _filter_scratch.data(), selected_rows == 0, column, + _type, false, &rows_read)); + advance_selected_span(rows_read); + if (column->size() != old_size + selected_rows) { + return Status::Corruption( + "Native parquet selection appended {} rows, expected {} for column {}", + column->size() - old_size, selected_rows, _name); + } + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + record_page_fragments(sync_native_profile()); + return Status::OK(); +} + +Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + *used_filter = false; + row_filter->clear(); + if (!_dictionary_filter_enabled) { + return Status::OK(); + } + *used_filter = true; + + _filter_scratch.assign(static_cast(batch_rows), 0); + for (uint16_t idx = 0; idx < selected_rows; ++idx) { + _filter_scratch[selection.get_index(idx)] = 1; + } + const bool nullable = _type->is_nullable(); + DataTypePtr id_type = std::make_shared(); + if (nullable) { + id_type = make_nullable(id_type); + } + if (!_dictionary_id_column) { + _dictionary_id_column = id_type->create_column(); + } + _dictionary_id_column->clear(); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_filter(batch_rows, _filter_scratch.data(), selected_rows == 0, + _dictionary_id_column, id_type, true, &rows_read)); + advance_selected_span(rows_read); + if (_dictionary_id_column->size() != selected_rows) { + return Status::Corruption( + "Native parquet dictionary reader appended {} rows, expected {} for {}", + _dictionary_id_column->size(), selected_rows, _name); + } + + const ColumnInt32* ids = nullptr; + const NullMap* null_map = nullptr; + if (const auto* nullable_ids = check_and_get_column(*_dictionary_id_column); + nullable_ids != nullptr) { + ids = check_and_get_column(nullable_ids->get_nested_column()); + null_map = &nullable_ids->get_null_map_data(); + } else { + ids = check_and_get_column(*_dictionary_id_column); + } + DORIS_CHECK(ids != nullptr); + + if (!_matched_dictionary_ids) { + _matched_dictionary_ids = ColumnInt32::create(); + } + _matched_dictionary_ids->clear(); + auto& matched_ids = assert_cast(*_matched_dictionary_ids).get_data(); + row_filter->reserve(selected_rows); + const auto& id_data = ids->get_data(); + for (size_t row = 0; row < selected_rows; ++row) { + bool keep = false; + if (null_map == nullptr || (*null_map)[row] == 0) { + const int32_t dictionary_id = id_data[row]; + if (dictionary_id < 0 || + static_cast(dictionary_id) >= dictionary_filter.size()) { + return Status::Corruption( + "Invalid parquet dictionary id {} for column {} with {} entries", + dictionary_id, _name, dictionary_filter.size()); + } + keep = dictionary_filter[static_cast(dictionary_id)] != 0; + if (keep) { + matched_ids.push_back(dictionary_id); + } + } + row_filter->push_back(keep ? 1 : 0); + } + + const auto* matched_id_column = check_and_get_column(*_matched_dictionary_ids); + DORIS_CHECK(matched_id_column != nullptr); + auto string_values = + DORIS_TRY(_native_reader->convert_dict_column_to_string_column(matched_id_column)); + RETURN_IF_ERROR(append_non_null_dictionary_values(column, std::move(string_values))); + if (_profile.reader_select_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); + } + update_reader_read_rows(cast_set(matched_ids.size())); + update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); + record_page_fragments(sync_native_profile()); + return Status::OK(); +} + +void NativeColumnReader::record_page_fragments(int64_t page_fragments) { + if (_profile.native_page_fragments != nullptr) { + COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); + } + if (page_fragments > 1 && _profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_profile.page_crossing_batches, 1); + } +} + +int64_t NativeColumnReader::sync_native_profile() { + if (_native_reader == nullptr) { + return 0; + } + const auto stats = _native_reader->column_statistics(); + const auto& reported = _reported_native_stats; + const auto& last_query = _last_native_query_stats; + if (_profile.decompress_time != nullptr) { + COUNTER_UPDATE(_profile.decompress_time, stats.decompress_time - reported.decompress_time); + } + if (_profile.decompress_count != nullptr) { + COUNTER_UPDATE(_profile.decompress_count, stats.decompress_cnt - reported.decompress_cnt); + } + if (_profile.decode_header_time != nullptr) { + COUNTER_UPDATE(_profile.decode_header_time, + stats.decode_header_time - reported.decode_header_time); + } + if (_profile.decode_value_time != nullptr) { + COUNTER_UPDATE(_profile.decode_value_time, + stats.decode_value_time - reported.decode_value_time); + } + if (_profile.decode_dictionary_time != nullptr) { + COUNTER_UPDATE(_profile.decode_dictionary_time, + stats.decode_dict_time - reported.decode_dict_time); + } + if (_profile.decode_level_time != nullptr) { + COUNTER_UPDATE(_profile.decode_level_time, + stats.decode_level_time - reported.decode_level_time); + } + if (_profile.decode_null_map_time != nullptr) { + COUNTER_UPDATE(_profile.decode_null_map_time, + stats.decode_null_map_time - reported.decode_null_map_time); + } + if (_profile.convert_time != nullptr) { + COUNTER_UPDATE(_profile.convert_time, stats.convert_time - reported.convert_time); + } + if (_profile.materialization_time != nullptr) { + COUNTER_UPDATE(_profile.materialization_time, stats.convert_time - reported.convert_time); + } + if (_profile.page_index_read_calls != nullptr) { + COUNTER_UPDATE(_profile.page_index_read_calls, + stats.page_index_read_calls - reported.page_index_read_calls); + } + if (_profile.skip_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.skip_page_header_count, + stats.skip_page_header_num - reported.skip_page_header_num); + } + if (_profile.parse_page_header_count != nullptr) { + COUNTER_UPDATE(_profile.parse_page_header_count, + stats.parse_page_header_num - reported.parse_page_header_num); + } + if (_profile.read_page_header_time != nullptr) { + COUNTER_UPDATE(_profile.read_page_header_time, + stats.read_page_header_time - reported.read_page_header_time); + } + // chunk_statistics() adds the PageReader's cumulative counters into its own accumulator on + // every query. The difference between two raw query results is therefore the PageReader's + // current cumulative value, not the increment since the previous query. +#define RECONSTRUCT_PAGE_STAT(field) (stats.field - last_query.field) + const int64_t page_read_count = RECONSTRUCT_PAGE_STAT(page_read_counter); + const int64_t page_read_delta = page_read_count - reported.page_read_counter; + if (_profile.page_read_count != nullptr) { + COUNTER_UPDATE(_profile.page_read_count, page_read_delta); + } + if (_profile.page_cache_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_write_count, + stats.page_cache_write_counter - reported.page_cache_write_counter); + } + if (_profile.page_cache_compressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_write_count, + stats.page_cache_compressed_write_counter - + reported.page_cache_compressed_write_counter); + } + if (_profile.page_cache_decompressed_write_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_write_count, + stats.page_cache_decompressed_write_counter - + reported.page_cache_decompressed_write_counter); + } + if (_profile.page_cache_hit_count != nullptr) { + COUNTER_UPDATE( + _profile.page_cache_hit_count, + RECONSTRUCT_PAGE_STAT(page_cache_hit_counter) - reported.page_cache_hit_counter); + } + if (_profile.page_cache_miss_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_miss_count, + RECONSTRUCT_PAGE_STAT(page_cache_missing_counter) - + reported.page_cache_missing_counter); + } + if (_profile.page_cache_compressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_compressed_hit_count, + RECONSTRUCT_PAGE_STAT(page_cache_compressed_hit_counter) - + reported.page_cache_compressed_hit_counter); + } + if (_profile.page_cache_decompressed_hit_count != nullptr) { + COUNTER_UPDATE(_profile.page_cache_decompressed_hit_count, + RECONSTRUCT_PAGE_STAT(page_cache_decompressed_hit_counter) - + reported.page_cache_decompressed_hit_counter); + } +#undef RECONSTRUCT_PAGE_STAT + _reported_native_stats = stats; + _reported_native_stats.page_read_counter = page_read_count; + _reported_native_stats.page_cache_hit_counter = + stats.page_cache_hit_counter - last_query.page_cache_hit_counter; + _reported_native_stats.page_cache_missing_counter = + stats.page_cache_missing_counter - last_query.page_cache_missing_counter; + _reported_native_stats.page_cache_compressed_hit_counter = + stats.page_cache_compressed_hit_counter - last_query.page_cache_compressed_hit_counter; + _reported_native_stats.page_cache_decompressed_hit_counter = + stats.page_cache_decompressed_hit_counter - + last_query.page_cache_decompressed_hit_counter; + _last_native_query_stats = stats; + return page_read_delta; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h new file mode 100644 index 00000000000000..8b66afac81d01c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +#include "format/parquet/parquet_common.h" +#include "format/parquet/vparquet_column_reader.h" +#include "format/table/table_schema_change_helper.h" +#include "format_v2/parquet/reader/column_reader.h" + +namespace doris { +class FileMetaData; +class RuntimeState; +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet { + +// Production adapter from FileScannerV2's selection-oriented reader contract to Doris' native +// Parquet page/encoding reader. The owned native reader decodes page bytes directly into the final +// Doris column. It never creates an Arrow Array/Builder, DecodedColumnView, or intermediate nested +// values_column. +// +// Cursor contract: +// - read(rows) consumes and appends exactly `rows` logical top-level rows; +// - select(selection, batch_rows) consumes `batch_rows` and appends only selected rows; +// - skip(rows) consumes `rows` with an all-false FilterMap and appends no payload; +// - one adapter lives for one top-level column in one Row Group, so decoder dictionaries, +// decompression buffers, level buffers, converters, and destination capacity survive adaptive +// batch-size changes. +class NativeColumnReader final : public ParquetColumnReader { +public: + static Status create(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, io::FileReaderSPtr file, + const FileMetaData* metadata, int row_group_id, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, + RuntimeState* runtime_state, bool enable_page_cache, + bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + ~NativeColumnReader() override; + + Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; + Status skip(int64_t rows) override; + Status select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + MutableColumnPtr& column) override; + Status select_with_dictionary_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, IColumn::Filter* row_filter, + bool* used_filter) override; + +private: + NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, + ParquetColumnReaderProfile profile); + + Status init(io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, + FieldSchema* field, std::shared_ptr schema_node, + std::set projected_column_ids, + const std::vector& selected_ranges, + const std::unordered_map& offset_indexes, + const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, + bool enable_page_cache, bool enable_dictionary_filter); + + Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + MutableColumnPtr& column, const DataTypePtr& output_type, + bool dictionary_ids, int64_t* rows_read); + int64_t sync_native_profile(); + void record_page_fragments(int64_t page_fragments); + Status validate_selected_span(int64_t rows); + void advance_selected_span(int64_t rows); + + // Native ParquetColumnReader keeps a reference to RowRanges; declare it before the reader. + ::doris::RowRanges _row_ranges; + std::set _projected_column_ids; + std::set _filter_column_ids; + std::unordered_map _offset_indexes; + std::shared_ptr _schema_node; + std::unique_ptr<::doris::ParquetColumnReader> _native_reader; + std::unique_ptr _page_cache_runtime_state; + std::vector _selected_ranges; + size_t _selected_range_idx = 0; + int64_t _logical_row_position = 0; + int64_t _row_group_rows = 0; + + bool _dictionary_filter_enabled = false; + bool _nested = false; + // Most native statistics are ordinary cumulative values. Page/cache statistics are special: + // v1 folds the PageReader's cumulative snapshot into ColumnChunkReader on every query. Keep the + // previous raw query so sync_native_profile() can reconstruct the real cumulative page value + // instead of reporting the same pages once per FileScannerV2 batch. + ::doris::ParquetColumnReader::ColumnStatistics _last_native_query_stats; + ::doris::ParquetColumnReader::ColumnStatistics _reported_native_stats; + std::vector _filter_scratch; + MutableColumnPtr _skip_column; + MutableColumnPtr _dictionary_id_column; + MutableColumnPtr _matched_dictionary_ids; +}; + +} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 7402595fc37d06..482913c6ff015b 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -37,8 +37,12 @@ #include "core/assert_cast.h" #include "core/block/block.h" +#include "core/column/column_array.h" +#include "core/column/column_decimal.h" +#include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_string.h" +#include "core/column/column_struct.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" @@ -475,6 +479,27 @@ std::shared_ptr build_timestamp_array(const std::shared_ptr build_decimal_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); + for (const auto value : values) { + EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); + } + return finish_array(&builder); +} + +std::shared_ptr build_fixed_binary_array(const std::shared_ptr& type, + const std::vector& values) { + arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); + const int32_t byte_width = + std::static_pointer_cast(type)->byte_width(); + for (const auto& value : values) { + EXPECT_EQ(value.size(), byte_width); + EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_struct_array(const std::vector& ids, const std::vector& names) { auto struct_type = arrow::struct_({arrow::field("id", arrow::int32(), false), @@ -517,6 +542,29 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_decimal_and_fixed_binary_parquet_file(const std::string& file_path) { + auto decimal_type = arrow::decimal128(38, 6); + auto fixed_type = arrow::fixed_size_binary(4); + auto schema = arrow::schema({arrow::field("decimal_value", decimal_type, false), + arrow::field("fixed_value", fixed_type, false)}); + auto table = arrow::Table::Make( + schema, + {build_decimal_array(decimal_type, {1234567, -1, 0, -987654321, 42}), + build_fixed_binary_array(fixed_type, {"ABCD", std::string("\0x\0y", 4), "wxyz", "1234", + std::string("\xff\x00\x7f\x80", 4)})}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.disable_dictionary(); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 2, + builder.build())); +} + std::shared_ptr build_nullable_int_string_map_array() { auto key_builder = std::make_shared(); auto value_builder = std::make_shared(); @@ -686,6 +734,27 @@ void write_nullable_string_struct_parquet_file(const std::string& file_path) { ROW_COUNT, builder.build())); } +void write_nullable_complex_parquet_file(const std::string& file_path) { + auto map_array = build_nullable_int_string_map_array(); + auto list_array = build_nullable_string_list_array(); + auto struct_array = build_nullable_string_struct_array(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("m", map_array->type(), true), + arrow::field("a", list_array->type(), true), + arrow::field("s", struct_array->type(), true)}), + {map_array, list_array, struct_array}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_nullable_struct_with_list_parquet_file(const std::string& file_path) { auto scalar_first = build_nullable_struct_with_list_array(false); auto list_first = build_nullable_struct_with_list_array(true); @@ -1271,6 +1340,152 @@ TEST_F(NewParquetReaderTest, CountStructWithRepeatedChildUsesTopLevelRowBoundari } } +TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchChanges) { + write_nullable_complex_parquet_file(_file_path); + RuntimeProfile profile("native_complex_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 3); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1), + field_projection(2)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + int batch = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + if (rows == 0) { + continue; + } + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + if (++batch == 1) { + // Adaptive sizing changes only the logical row cap. The persistent native readers and + // their level/string scratch must continue from the same page cursors. + reader->set_batch_size(3); + } + } + + const auto& nullable_map = assert_cast(*output[0]); + ASSERT_EQ(nullable_map.size(), ROW_COUNT); + EXPECT_FALSE(nullable_map.is_null_at(0)); + EXPECT_TRUE(nullable_map.is_null_at(1)); + EXPECT_FALSE(nullable_map.is_null_at(2)); + const auto& map = assert_cast(nullable_map.get_nested_column()); + EXPECT_EQ(map.get_offsets(), ColumnArray::Offsets64({1, 1, 1, 2, 3})); + const auto& map_keys = assert_cast(map.get_keys()); + const auto& key_values = assert_cast(map_keys.get_nested_column()); + ASSERT_EQ(key_values.size(), 3); + EXPECT_EQ(key_values.get_element(0), 10); + EXPECT_EQ(key_values.get_element(2), 30); + const auto& map_values = assert_cast(map.get_values()); + const auto& value_strings = assert_cast(map_values.get_nested_column()); + EXPECT_EQ(value_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(value_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(map_values.is_null_at(2)); + + const auto& nullable_array = assert_cast(*output[1]); + ASSERT_EQ(nullable_array.size(), ROW_COUNT); + EXPECT_TRUE(nullable_array.is_null_at(1)); + const auto& array = assert_cast(nullable_array.get_nested_column()); + EXPECT_EQ(array.get_offsets(), ColumnArray::Offsets64({2, 2, 2, 3, 4})); + const auto& array_values = assert_cast(array.get_data()); + const auto& array_strings = assert_cast(array_values.get_nested_column()); + EXPECT_EQ(array_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(array_strings.get_data_at(1).size, 4096); + EXPECT_TRUE(array_values.is_null_at(2)); + EXPECT_EQ(array_strings.get_data_at(3).size, 4096); + + const auto& nullable_struct = assert_cast(*output[2]); + ASSERT_EQ(nullable_struct.size(), ROW_COUNT); + EXPECT_TRUE(nullable_struct.is_null_at(1)); + const auto& struct_column = + assert_cast(nullable_struct.get_nested_column()); + const auto& payload = assert_cast(struct_column.get_column(0)); + const auto& payload_strings = assert_cast(payload.get_nested_column()); + EXPECT_EQ(payload_strings.get_data_at(0).to_string(), "small"); + EXPECT_EQ(payload_strings.get_data_at(2).size, 4096); + EXPECT_TRUE(payload.is_null_at(3)); + EXPECT_EQ(payload_strings.get_data_at(4).size, 4096); + const auto& ids = assert_cast(struct_column.get_column(1)); + const auto& id_values = assert_cast(ids.get_nested_column()); + EXPECT_EQ(id_values.get_element(0), 1); + EXPECT_EQ(id_values.get_element(4), 4); + + ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + EXPECT_EQ(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + ASSERT_NE(profile.get_counter("NestedBatches"), nullptr); + EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { + write_decimal_and_fixed_binary_parquet_file(_file_path); + RuntimeProfile profile("native_decimal_fixed_binary_materialization"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(2); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + ASSERT_TRUE(reader->open(request).ok()); + + MutableColumns output; + output.reserve(schema.size()); + for (const auto& field : schema) { + output.push_back(field.type->create_column()); + } + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + for (size_t column = 0; column < output.size(); ++column) { + output[column]->insert_range_from(*block.get_by_position(column).column, 0, rows); + } + reader->set_batch_size(3); + } + + const auto& decimals = assert_cast( + assert_cast(*output[0]).get_nested_column()); + ASSERT_EQ(decimals.size(), ROW_COUNT); + EXPECT_EQ(decimals.get_element(0), Decimal128V3(1234567)); + EXPECT_EQ(decimals.get_element(1), Decimal128V3(-1)); + EXPECT_EQ(decimals.get_element(3), Decimal128V3(-987654321)); + + const auto& fixed_values = assert_cast( + assert_cast(*output[1]).get_nested_column()); + ASSERT_EQ(fixed_values.size(), ROW_COUNT); + EXPECT_EQ(fixed_values.get_data_at(0).to_string(), "ABCD"); + EXPECT_EQ(fixed_values.get_data_at(1).to_string(), std::string("\0x\0y", 4)); + EXPECT_EQ(fixed_values.get_data_at(4).to_string(), std::string("\xff\x00\x7f\x80", 4)); + + ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + EXPECT_EQ(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("ConvertTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { write_struct_filter_parquet_file(_file_path); auto reader = create_reader(); @@ -1532,7 +1747,7 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); ASSERT_NE(profile.get_counter("PageCacheWriteCount"), nullptr); - EXPECT_EQ(profile.get_counter("PageReadCount")->value(), 0); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0); } @@ -1645,24 +1860,35 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterReadCalls"), nullptr); + ASSERT_NE(profile.get_counter("FileFooterHitCache"), nullptr); ASSERT_GT(profile.get_counter("FileReaderCreateTime")->value(), 0); EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 1); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); - EXPECT_GT(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("ArrowReadRecordsTime")->value(), 0); EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); + EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value() + + profile.get_counter("FileFooterHitCache")->value(), + 1); rows = 0; eof = false; @@ -1777,11 +2003,15 @@ TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RowsFilteredByConjunct"), nullptr); ASSERT_NE(profile.get_counter("TotalBatches"), nullptr); + ASSERT_NE(profile.get_counter("DenseBatches"), nullptr); + ASSERT_NE(profile.get_counter("SelectedBatches"), nullptr); ASSERT_NE(profile.get_counter("EmptySelectionBatches"), nullptr); EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 0); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); + EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); + EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 0); EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); } @@ -2177,8 +2407,14 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) EXPECT_EQ(values, std::vector({"az", "za"})); EXPECT_EQ(payloads, std::vector({20, 50})); EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); + // Dictionary probing finishes on the base Arrow reader before the native data-page path + // installs its row-group-scoped merge reader. The merge profile therefore describes only + // native projected chunk reads and cannot be perturbed by dictionary ReadAt order. ASSERT_NE(profile.get_counter("MergedIO"), nullptr); ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); + EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); + ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); + EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); } TEST_F(NewParquetReaderTest, DictionaryPredicateWorksWithoutRuntimeProfile) { @@ -2258,7 +2494,9 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateSkipsRemainingPredicateColumnsWh // second predicate column is skipped after the selection becomes empty, which verifies the // StarRocks-style round-by-round policy: only rows surviving previous predicates are read. EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 6); - EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 6); + // Five dictionary ids are rejected before the residual predicate; the remaining predicate + // reader then skips all six logical rows once the selection is empty. + EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 11); } TEST_F(NewParquetReaderTest, DictionaryPredicateRunsResidualConjunctOnSurvivors) { @@ -2547,7 +2785,8 @@ TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIn // Scenario: the selected range starts after page-index-pruned rows. The scheduler defers that range // gap for the non-predicate payload reader, then flushes it exactly once before materialization. The -// page skip plan advances the reader without calling Arrow SkipRecords or double-skipping row 64. +// native RowRanges/OffsetIndex plan advances both readers without decoding the rejected pages or +// double-skipping row 64. TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { write_page_index_filter_pair_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_page_skip"); @@ -2571,7 +2810,8 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { bool eof = false; while (!eof) { size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 6661406f1934b6..72b5e766052b61 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -69,27 +69,28 @@ planning is distinct from a persistent per-column reader, and page/encoding deco cursor-based contracts rather than an Arrow array as an intermediate result. The migration is deliberately incremental because the existing v1 native kernel already contains -mature Parquet page and encoding support: +mature Parquet page and encoding support. The first production step reuses that kernel behind a +v2-owned adapter; decoder logic is copied into the v2 tree only when v2 must diverge from it: | Stage | State and boundary | | --- | --- | -| Native encoding kernel | Reimplement the proven Doris v1 decoder algorithms under `be/src/format_v2/parquet/` behind a schema-independent physical Column Chunk contract. Selection-aware scalar decoding and reusable string scratch are the first vertical slice. | -| Native column reader | Make the leaf reader and its SerDe, null map, selection ranges, binary values, level buffers, and conversion scratch persistent for a Row Group. Add direct Doris materialization paths. | +| Native encoding kernel | The current adapter calls the unchanged Doris native kernel used by v1, preserving its complete type/encoding matrix and page-cache behavior. Any decoder change is reimplemented under `be/src/format_v2/parquet/`; v1 remains unchanged. | +| Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | | Complex reconstruction | Build one shared Dremel level plan per requested parent-row range and use it for STRUCT/ARRAY/MAP siblings, offsets, and null maps. | | Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | | Compatibility removal | Remove Arrow data-read adapters after type/encoding/page/writer compatibility and performance gates pass. Production v2 never falls back from a selected native reader to Arrow; an unsupported combination returns an explicit error. | -At the current migration boundary, Arrow is still present in the v2 footer/schema, planning, and -data path. Such a patch is transitional and must not be described as a completed native decoder. -The production target contains no Arrow object in the v2 runtime call chain. Arrow may be used only -as a test oracle while differential tests are being developed, never as a runtime compatibility -fallback. +At the current migration boundary, ordinary value scans no longer use Arrow `RecordReader`, arrays, +or builders. Arrow remains in footer/schema planning, dictionary probing, and the existing +levels-only aggregate path. It is not a runtime fallback: after an ordinary column selects the +native reader, decode errors are returned directly. The production target eventually removes the +remaining Arrow metadata/aggregate objects; tests may also use Arrow as a fixture writer or oracle. -All production integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the -correctness and performance control. Decoder code required by v2 is reimplemented in the v2 tree; -the migration does not change v1 or route v1 reads through a new v2 path. This separation makes -differential testing meaningful and prevents a v2 experiment from regressing the established -reader. +All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness +and performance control. Reusing its stable native kernel gives v2 the same physical/logical type, +encoding, malformed-input, conversion, and page-cache behavior without duplicating thousands of +lines before an actual semantic change is required. When such a change is required, the affected +decoder is reimplemented in the v2 tree instead of modifying v1. ### 2.2 Native Interface Ownership @@ -160,6 +161,13 @@ sequenceDiagram expose an Arrow adapter for metadata consumers, but native page decoding reads the same stable Doris byte ranges without producing Arrow arrays. +For every selected Row Group, dictionary/index probes finish on the metadata adapter first. The +scheduler then computes projected physical Column Chunk ranges and installs one shared native +`MergeRangeFileReader` when their average size is below v1's small-I/O threshold. All predicate and +lazy output readers share that wrapper; their internal `BufferedFileStreamReader` prefetch is +disabled in this mode, avoiding duplicate buffers. Large chunks and in-memory files use the base +reader, while remote FileCache prefetch remains the non-MergeRange path. + > Planning intentionally proceeds from cheap to expensive. Split and metadata pruning reduce the > candidate set before finer indexes are read for surviving Row Groups, avoiding index I/O for data > that is already known to be irrelevant. @@ -439,7 +447,7 @@ batches. #### Complex-reader interface and materialization cost -The Arrow migration adapter currently exposes four stateful operations: +The retained Arrow complex-reader facade exposes four stateful operations: `load_nested_batch()`, `load_nested_levels_batch()`, `build_nested_column()`, and `consume_nested_column()`. This split made shape-only reads possible while Arrow still owned value decoding, but it is not the target native interface. It has three measurable costs: @@ -478,8 +486,11 @@ leaf as the entry-shape owner and validates the value leaf against the same entr one representative leaf for parent validity and only checks sibling alignment while decoding each child. -During migration, the four Arrow-facing methods are temporary scaffolding, not a runtime fallback. -Their leaf reader, +Ordinary production scans do not call these four methods: `NativeColumnReader::read/select/skip` +uses the native `read_column_data()` boundary and materializes the complete complex column directly. +The facade remains for the unchanged levels-only aggregate path and focused control tests; it is not +a fallback after native reader selection. `ParquetLeafBatch` is the facade's decoded Arrow/level +container and is intentionally forbidden from the ordinary value path. Its leaf reader, SerDe, binary/null/level scratch, selection ranges, nested batches, parent nulls, entry counts, and child-column handles must be persistent so the facade does not add per-batch allocation churn. New native decoder code must not depend on this phase ordering or reproduce the temporary @@ -587,9 +598,11 @@ trustworthy version identity is unavailable, the footer is read and parsed witho reusable entry. Parse failures, short files, encrypted/unsupported metadata, and schema-affecting option changes cannot populate or reuse a successful entry. -During migration, serialized-footer caching and parsed-native-metadata caching may be enabled in -separate steps, but both use the same identity and lifetime rules. An Arrow-parsed metadata object -must not be treated as the native cache value or leak an Arrow lifetime into the native decoder. +V2 calls the same footer parser/cache and key builder as v1. On either a cache hit or miss, the +immutable native Thrift metadata is the owner. While Arrow planning remains, V2 serializes that +already cached Thrift object into Arrow's metadata parser, so opening the planner performs no second +footer read. The Arrow object is never the cache value and its lifetime does not enter the native +decoder. ### Page Cache Parity with V1 @@ -611,9 +624,9 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. -- When average projected chunks are small and the reader is not in-memory, install - MergeRangeFileReader so subsequent native range reads or transitional Arrow `ReadAt` calls use - merged reads. +- The remaining Arrow metadata/index path may use MergeRangeFileReader. The current native + BufferedFileStreamReader is not routed through that Arrow wrapper; it uses v1's stream/page cache + path while background prefetch warms the same Doris FileCache blocks. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. From e3048f121482f94f95d35f1b9fd303ba18314142 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 15:18:25 +0800 Subject: [PATCH 04/34] [refactor](be) Remove obsolete Parquet v2 Arrow value readers Issue Number: close #xxx Related PR: #65674 Problem Summary: The Parquet v2 native scan path no longer uses the prototype Arrow value-reader hierarchy, but its decoded leaf batch, nested load/build/consume protocol, factories, and tests remained reachable from a shape-only aggregate path. Remove that intermediate layer and isolate the unchanged COUNT(nullable_col) level-reading compatibility path behind a narrow API. Ordinary scans now expose only the persistent native read/skip/select contract. None - Test: Unit Test (remote validation pending) - Behavior changed: No - Does this need documentation: Yes (updated in this commit) --- be/src/format_v2/AGENTS.md | 28 +- .../parquet/parquet_column_schema.cpp | 2 +- .../format_v2/parquet/parquet_column_schema.h | 8 +- be/src/format_v2/parquet/parquet_profile.h | 8 +- be/src/format_v2/parquet/parquet_reader.cpp | 43 +- be/src/format_v2/parquet/parquet_scan.cpp | 26 +- be/src/format_v2/parquet/parquet_type.cpp | 25 - be/src/format_v2/parquet/parquet_type.h | 3 - .../parquet/reader/column_reader.cpp | 600 +-- .../format_v2/parquet/reader/column_reader.h | 201 +- .../parquet/reader/count_column_reader.cpp | 247 ++ .../parquet/reader/count_column_reader.h | 76 + .../parquet/reader/list_column_reader.cpp | 230 - .../parquet/reader/list_column_reader.h | 61 - .../parquet/reader/map_column_reader.cpp | 285 -- .../parquet/reader/map_column_reader.h | 66 - .../parquet/reader/native_column_reader.cpp | 2 +- .../parquet/reader/native_column_reader.h | 1 + .../reader/nested_column_materializer.cpp | 70 - .../reader/nested_column_materializer.h | 45 - .../parquet/reader/parquet_leaf_reader.cpp | 850 ---- .../parquet/reader/parquet_leaf_reader.h | 213 - .../reader/row_position_column_reader.cpp | 1 + .../parquet/reader/scalar_column_reader.cpp | 596 --- .../parquet/reader/scalar_column_reader.h | 118 - .../parquet/reader/struct_column_reader.cpp | 287 -- .../parquet/reader/struct_column_reader.h | 70 - be/src/format_v2/parquet/selection_vector.h | 6 +- .../parquet/parquet_column_reader_test.cpp | 3682 ----------------- .../parquet/parquet_leaf_reader_test.cpp | 535 --- .../parquet/parquet_reader_control_test.cpp | 1125 +---- .../parquet/parquet_serde_reader_test.cpp | 459 -- .../format_v2/parquet/parquet_type_test.cpp | 6 - docs/file-scanner-v2-parquet-scan-design.md | 75 +- 34 files changed, 459 insertions(+), 9591 deletions(-) create mode 100644 be/src/format_v2/parquet/reader/count_column_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/count_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/list_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/list_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/map_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/map_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/nested_column_materializer.cpp delete mode 100644 be/src/format_v2/parquet/reader/nested_column_materializer.h delete mode 100644 be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/parquet_leaf_reader.h delete mode 100644 be/src/format_v2/parquet/reader/scalar_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/scalar_column_reader.h delete mode 100644 be/src/format_v2/parquet/reader/struct_column_reader.cpp delete mode 100644 be/src/format_v2/parquet/reader/struct_column_reader.h delete mode 100644 be/test/format_v2/parquet/parquet_column_reader_test.cpp delete mode 100644 be/test/format_v2/parquet/parquet_leaf_reader_test.cpp delete mode 100644 be/test/format_v2/parquet/parquet_serde_reader_test.cpp diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 5b6007550fb94b..c669de184076c6 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -131,16 +131,16 @@ instructions as well; this file adds format-v2-specific review expectations. shared definition/repetition-level plan that identifies parent-row boundaries, empty and null collections, null ancestors, and rows spanning pages. Sibling readers in a STRUCT, ARRAY, or MAP must consume the same parent-row plan rather than independently inferring offsets or null maps. -- Treat the current `load_nested_batch()` / `load_nested_levels_batch()` / - `build_nested_column()` / `consume_nested_column()` split as an Arrow-migration facade, not the - native kernel API. Review new native code for one explicit nested-read request carrying parent - row count, selection, and `VALUES` versus `LEVELS_ONLY` mode. Shape parsing, payload-cursor - advancement, validation, and output must have an obvious owner; callers must not depend on an - undocumented load-before-build phase. -- `ParquetLeafBatch` belongs only to that isolated Arrow facade and its levels-only migration path. - Ordinary predicate/output scans must construct `NativeColumnReader` and may not route decoded - values through `ParquetLeafBatch`, `DecodedColumnView`, Arrow arrays, or temporary nested Doris - columns before the final destination column. +- The old Arrow value-reader hierarchy (`ParquetLeafBatch`, scalar/list/map/struct readers, and the + nested load/build/consume protocol) has been removed. Do not reintroduce an intermediate decoded + batch or a stateful load-before-build phase. Ordinary predicate/output scans construct + `NativeColumnReader`, consume compressed page data through the native decoder, and append directly + into the final Doris column. +- `CountColumnReader` is the sole Arrow data-page exception. It is an isolated shape-only adapter + for the existing `COUNT(nullable_col)` pushdown because Arrow exposes no independent public level + decoder. It selects one representative leaf (the key for MAP), copies only definition/repetition + levels, immediately releases binary builder chunks, and exposes no value API. It must not be + reused as a scan reader or expanded into a fallback path. - Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one level traversal and share the result. For example, `[[1, 2], NULL, []]` must yield entry counts `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning the same levels per child. MAP key @@ -169,10 +169,10 @@ instructions as well; this file adds format-v2-specific review expectations. and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. A direct path may not bypass the conversion rules used by the general conversion path. -- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, unsupported - physical types, encodings, page layouts, or malformed inputs return an explicit status. Arrow may - be used by the explicitly documented metadata-planning or levels-only migration paths and as a - test oracle; no Arrow array, builder, RecordReader, or metadata lifetime belongs in ordinary +- Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, + unsupported physical types, encodings, page layouts, or malformed inputs return an explicit + status. Arrow may be used by metadata planning, the isolated COUNT shape adapter, and as a test + oracle; no Arrow array, builder, RecordReader, or metadata lifetime belongs in ordinary value materialization. - Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across batches. String-like decoders should gather selected `StringRef` values and append once per batch, diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index 1cdfed80bd273b..59a315096cd92a 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -264,7 +264,7 @@ Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, // optional binary value (STRING); // } // } - // Accept that schema here so compatible files can be read. MapColumnReader validates the + // Accept that schema here so compatible files can be read. The native reader validates the // materialized key column and rejects data that really contains null map keys. result->entry_group = &entry_group; result->entry_context = child_context(map_context, entry_node, 0); diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index 1fb7262aabde6f..ecacdfe1d97c4c 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -31,10 +31,10 @@ class SchemaDescriptor; namespace doris::format::parquet { enum class ParquetColumnSchemaKind { - PRIMITIVE, // primitive leaf -> ScalarColumnReader - STRUCT, // struct -> StructColumnReader - LIST, // array -> ListColumnReader - MAP, // map -> MapColumnReader + PRIMITIVE, // physical primitive leaf + STRUCT, // Parquet group with STRUCT semantics + LIST, // Parquet group with LIST semantics + MAP, // Parquet group with MAP semantics }; // ============================================================================ diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index e1b8abee3dff5c..f0cdfd4d00ab66 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -34,8 +34,9 @@ struct ParquetColumnReaderProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() - RuntimeProfile::Counter* arrow_read_records_time = nullptr; // Arrow RecordReader time (ns) - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; // Arrow SkipRecords time (ns) + // COUNT(nullable_col) shape-only compatibility path; ordinary scans keep both counters zero. + RuntimeProfile::Counter* arrow_read_records_time = nullptr; + RuntimeProfile::Counter* arrow_skip_records_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) // Native page/encoding reader internals. These counters intentionally mirror v1 so a v1/v2 // profile comparison attributes page IO, decompression, levels, value decode and conversion to @@ -69,7 +70,7 @@ struct ParquetColumnReaderProfile { // ============================================================================ // ============================================================================ struct ParquetScanProfile { - RuntimeProfile::Counter* raw_rows_read = nullptr; // raw rows read from RecordReader + RuntimeProfile::Counter* raw_rows_read = nullptr; // logical rows consumed before filtering RuntimeProfile::Counter* selected_rows = nullptr; // rows selected after conjunct filtering RuntimeProfile::Counter* rows_filtered_by_conjunct = nullptr; // rows filtered by conjuncts RuntimeProfile::Counter* lazy_read_filtered_rows = @@ -135,6 +136,7 @@ struct ParquetProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; RuntimeProfile::Counter* reader_skip_rows = nullptr; RuntimeProfile::Counter* reader_select_rows = nullptr; + // COUNT(nullable_col) shape-only compatibility path; ordinary scans keep these zero. RuntimeProfile::Counter* arrow_read_records_time = nullptr; RuntimeProfile::Counter* arrow_skip_records_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 03fe9ef50d00f0..6cc4f69277fb45 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -39,7 +39,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/parquet_statistics.h" -#include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/count_column_reader.h" #include "io/io_common.h" #include "runtime/runtime_state.h" @@ -213,11 +213,11 @@ const ParquetColumnSchema& projected_root_schema( } int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, - const ParquetColumnReader& shape_reader, + const CountColumnReader& shape_reader, int64_t expected_rows) { - const auto& def_levels = shape_reader.nested_definition_levels(); - const auto& rep_levels = shape_reader.nested_repetition_levels(); - const int64_t levels_written = shape_reader.nested_levels_written(); + const auto& def_levels = shape_reader.definition_levels(); + const auto& rep_levels = shape_reader.repetition_levels(); + const int64_t levels_written = shape_reader.levels_written(); DORIS_CHECK(levels_written >= expected_rows); if (root_schema.max_repetition_level == 0) { DORIS_CHECK(levels_written == expected_rows); @@ -728,14 +728,10 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r row_group_plan.row_group_id, e.what()); } - ParquetColumnReaderFactory column_reader_factory( - row_group, _state->file_context.schema->num_columns(), - &row_group_plan.page_skip_plans, _parquet_profile.page_skip_profile(), - _state->timezone, _state->enable_strict_mode, - _parquet_profile.scan_profile().column_reader_profile); - std::unique_ptr shape_reader; - RETURN_IF_ERROR(column_reader_factory.create_count_shape_reader( - root_schema, &count_projection, &shape_reader)); + std::unique_ptr shape_reader; + RETURN_IF_ERROR(CountColumnReader::create( + row_group, root_schema, &count_projection, + _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); int64_t row_group_cursor = 0; @@ -749,18 +745,19 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r while (range_rows_read < selected_range.length) { const int64_t batch_rows = std::min(_batch_size, selected_range.length - range_rows_read); - // COUNT(col) only needs the top-level NULL state. The shape reader loads - // def/rep levels from one representative leaf and does not build value_indices - // or values_column. MAP chooses the key leaf; ARRAY/STRUCT may choose a string - // leaf, but the levels-only protocol still avoids Doris-side string - // materialization for that leaf. + int64_t rows_read = 0; RETURN_IF_ERROR(_stop_status_if_requested( - shape_reader->load_nested_levels_batch(batch_rows))); - _record_scan_rows(batch_rows); + shape_reader->read_levels(batch_rows, &rows_read))); + if (rows_read != batch_rows) { + return Status::Corruption( + "Parquet COUNT reader returned {} rows, expected {}", rows_read, + batch_rows); + } + _record_scan_rows(rows_read); result->count += - count_loaded_non_null_values(root_schema, *shape_reader, batch_rows); - range_rows_read += batch_rows; - row_group_cursor += batch_rows; + count_loaded_non_null_values(root_schema, *shape_reader, rows_read); + range_rows_read += rows_read; + row_group_cursor += rows_read; } } } diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 5d09a189404bec..8880bf4ccaec73 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -35,7 +35,9 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" +#include "format_v2/parquet/reader/global_rowid_column_reader.h" #include "format_v2/parquet/reader/native_column_reader.h" +#include "format_v2/parquet/reader/row_position_column_reader.h" namespace doris::format::parquet { @@ -781,23 +783,18 @@ Status ParquetScanScheduler::open_next_row_group( native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, _merge_read_slice_size); - ParquetColumnReaderFactory column_reader_factory( - nullptr, file_context.schema->num_columns(), &row_group_plan.page_skip_plans, - _page_skip_profile, _timezone, _enable_strict_mode, - _scan_profile.column_reader_profile); for (const auto& col : request.predicate_columns) { const auto local_id = col.local_id(); if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + _current_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + _current_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } @@ -827,15 +824,16 @@ Status ParquetScanScheduler::open_next_row_group( } if (local_id == format::ROW_POSITION_COLUMN_ID) { _current_non_predicate_columns[local_id] = - column_reader_factory.create_row_position_column_reader( - _current_row_group_first_row); + std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { DORIS_CHECK(_global_rowid_context.has_value()); _current_non_predicate_columns[local_id] = - column_reader_factory.create_global_rowid_column_reader( - *_global_rowid_context, _current_row_group_first_row); + std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index 8411d53f0fba91..462908c119dedf 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -255,22 +255,6 @@ DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* colum return nullable ? make_nullable(type) : type; } -bool record_reader_physical_type_supported(::parquet::Type::type physical_type) { - switch (physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::INT96: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return true; - default: - return false; - } -} - } // namespace std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { @@ -299,13 +283,11 @@ ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* co result.doris_type = logical_type; } else if (!result.unsupported_reason.empty()) { result.doris_type = nullptr; - result.supports_record_reader = false; } else if (auto converted_type = converted_type_to_doris_type(column, &result); converted_type != nullptr) { result.doris_type = converted_type; } else if (!result.unsupported_reason.empty()) { result.doris_type = nullptr; - result.supports_record_reader = false; } else { result.doris_type = result.physical_doris_type; if (result.physical_type == ::parquet::Type::INT96) { @@ -318,16 +300,9 @@ ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* co (result.physical_type == ::parquet::Type::BYTE_ARRAY || result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - if (!record_reader_physical_type_supported(result.physical_type)) { - result.supports_record_reader = false; - } return result; } -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor) { - return type_descriptor.supports_record_reader; -} - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { switch (type_descriptor.physical_type) { case ::parquet::Type::BOOLEAN: diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index a4d99abc0e982a..feded03ccf25fc 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -70,7 +70,6 @@ struct ParquetTypeDescriptor { bool is_timestamp = false; // whether this is a timestamp type bool timestamp_is_adjusted_to_utc = false; // whether the timestamp is UTC-normalized bool is_string_like = false; // binary type that is neither decimal nor FLOAT16 - bool supports_record_reader = true; // whether Arrow RecordReader can read this type std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported }; @@ -78,8 +77,6 @@ std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); -bool supports_record_reader(const ParquetTypeDescriptor& type_descriptor); - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 352fbbd7c3d215..56c705fa12e518 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -15,145 +15,25 @@ #include "format_v2/parquet/reader/column_reader.h" -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include #include -#include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "runtime/runtime_profile.h" namespace doris::format::parquet { -namespace { - -class DataPageSkipFilter { -public: - DataPageSkipFilter(const ParquetPageSkipPlan* page_skip_plan, - ParquetPageSkipProfile page_skip_profile) - : _page_skip_plan(page_skip_plan), _page_skip_profile(page_skip_profile) { - DORIS_CHECK(_page_skip_plan != nullptr); - } - - bool operator()(const ::parquet::DataPageStats&) { - // Arrow invokes this callback once for each DATA_PAGE/DATA_PAGE_V2 and never for - // dictionary pages, so this ordinal matches Parquet OffsetIndex page locations. - const size_t page_idx = _next_data_page_idx++; - const bool skip = _page_skip_plan->should_skip_page(page_idx); - if (!skip) { - return false; - } - update_skip_profile(page_idx); - return true; - } - -private: - void update_skip_profile(size_t page_idx) const { - if (_page_skip_profile.skipped_pages != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_pages, 1); - } - if (_page_skip_profile.skipped_bytes != nullptr) { - COUNTER_UPDATE(_page_skip_profile.skipped_bytes, - _page_skip_plan->skipped_page_compressed_size(page_idx)); - } - } - - const ParquetPageSkipPlan* _page_skip_plan = nullptr; - ParquetPageSkipProfile _page_skip_profile; - size_t _next_data_page_idx = 0; -}; - -const ParquetPageSkipPlan* find_page_skip_plan( - const std::map* page_skip_plans, int leaf_column_id) { - if (page_skip_plans == nullptr) { - return nullptr; - } - const auto plan_it = page_skip_plans->find(leaf_column_id); - return plan_it == page_skip_plans->end() ? nullptr : &plan_it->second; -} - -void install_data_page_filter(std::unique_ptr<::parquet::PageReader>& page_reader, - const std::map* page_skip_plans, - int leaf_column_id, ParquetPageSkipProfile page_skip_profile) { - DORIS_CHECK(page_reader != nullptr); - const ParquetPageSkipPlan* page_skip_plan = - find_page_skip_plan(page_skip_plans, leaf_column_id); - if (page_skip_plan == nullptr) { - return; - } - page_reader->set_data_page_filter(DataPageSkipFilter(page_skip_plan, page_skip_profile)); -} - -bool supports_nested_scalar_record_reader(const ParquetColumnSchema& column_schema) { - if (column_schema.type_descriptor.supports_record_reader) { - return true; - } - const auto& type_descriptor = column_schema.type_descriptor; - if ((type_descriptor.extra_type_info != ParquetExtraTypeInfo::NONE && - type_descriptor.extra_type_info != ParquetExtraTypeInfo::FLOAT16) || - type_descriptor.is_decimal || type_descriptor.is_timestamp || - type_descriptor.is_string_like) { - return false; - } - if (type_descriptor.converted_type != ::parquet::ConvertedType::NONE && - type_descriptor.converted_type != ::parquet::ConvertedType::UNDEFINED) { - return false; - } - switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - return true; - default: - return false; - } - return true; -} -} // namespace +ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, + ParquetColumnReaderProfile profile) + : _profile(profile), + _field_id(schema.local_id), + _leaf_column_id(schema.leaf_column_id), + _type(std::move(type)), + _name(schema.name) {} Status ParquetColumnReader::skip(int64_t rows) { return Status::NotSupported("Parquet column skip is not implemented, rows={}", rows); } -void ParquetColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - int64_t child_cursor = nested_build_level_cursor(); - const auto& child_rep_levels = nested_repetition_levels(); - const int64_t child_levels_written = nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!is_or_has_repeated_child() || child_rep_level <= parent_repetition_level) { - break; - } - } - set_nested_build_level_cursor(child_cursor); -} - void ParquetColumnReader::update_reader_read_rows(int64_t rows) const { if (_profile.reader_read_rows != nullptr) { COUNTER_UPDATE(_profile.reader_read_rows, rows); @@ -166,21 +46,16 @@ void ParquetColumnReader::update_reader_skip_rows(int64_t rows) const { } } -Status ParquetColumnReader::select(const SelectionVector& sel, uint16_t selected_rows, +Status ParquetColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, MutableColumnPtr& column) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Parquet selected read result is null for column {}", - name()); - } - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); + DORIS_CHECK(column); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); - const auto ranges = selection_to_ranges(sel, selected_rows); + const auto ranges = selection_to_ranges(selection, selected_rows); int64_t cursor = 0; for (const auto& range : ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument("Invalid parquet selection range [{}, {}) for column {}", - range.start, range.start + range.length, name()); - } + DORIS_CHECK(range.start >= cursor); + DORIS_CHECK(range.start + range.length <= batch_rows); RETURN_IF_ERROR(skip(range.start - cursor)); int64_t range_rows_read = 0; @@ -206,453 +81,4 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& name()); } -ParquetColumnReaderFactory::ParquetColumnReaderFactory( - std::shared_ptr<::parquet::RowGroupReader> row_group, int num_leaf_columns, - const std::map* page_skip_plans, - ParquetPageSkipProfile page_skip_profile, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile column_reader_profile) - : _row_group(std::move(row_group)), - _record_readers(static_cast(num_leaf_columns)), - _dictionary_record_readers(static_cast(num_leaf_columns)), - _page_skip_plans(page_skip_plans), - _page_skip_profile(page_skip_profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _column_reader_profile(column_reader_profile) {} - -std::unique_ptr ParquetColumnReaderFactory::create_row_position_column_reader( - int64_t row_group_first_row) const { - return std::make_unique(row_group_first_row, _column_reader_profile); -} - -std::unique_ptr ParquetColumnReaderFactory::create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const { - return std::make_unique(context, row_group_first_row, - _column_reader_profile); -} - -Status ParquetColumnReaderFactory::make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, bool use_page_skip_plan, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - const auto* page_skip_plan = - use_page_skip_plan ? find_page_skip_plan(_page_skip_plans, column_schema.leaf_column_id) - : nullptr; - *reader = std::make_unique(column_schema, std::move(record_reader), - page_skip_plan, _timezone, _enable_strict_mode, - _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_scalar_column_reader( - const ParquetColumnSchema& column_schema, bool is_nested, bool read_dictionary, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (!column_schema.type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, - column_schema.type_descriptor.unsupported_reason); - } - if (is_nested && column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { - return Status::InvalidArgument("Parquet nested scalar reader requires primitive column {}", - column_schema.name); - } - if (column_schema.leaf_column_id < 0 || - column_schema.leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - column_schema.leaf_column_id, column_schema.name); - } - if (column_schema.descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", - column_schema.name); - } - if (!is_nested && (column_schema.descriptor->max_repetition_level() != 0 || - column_schema.descriptor->max_definition_level() > 1)) { - return Status::NotSupported( - "Current parquet scalar reader only supports flat primitive columns; column {} is " - "not supported", - column_schema.name); - } - if (is_nested && !supports_nested_scalar_record_reader(column_schema)) { - return Status::NotSupported( - "Current parquet nested scalar reader does not support column {}", - column_schema.name); - } - if (!is_nested && !column_schema.type_descriptor.supports_record_reader) { - return Status::NotSupported("Current parquet scalar reader does not support column {}", - column_schema.name); - } - std::shared_ptr<::parquet::internal::RecordReader> record_reader; - // Nested readers implement skip() by materializing rows into a scratch column. If Arrow - // page filtering is also installed, those scratch reads can consume the next selected row - // after a page-index range gap. Keep page filtering on flat scalar readers only. - RETURN_IF_ERROR(get_record_reader(column_schema.leaf_column_id, column_schema.descriptor, - column_schema.name, !is_nested, read_dictionary, - &record_reader)); - return make_scalar_column_reader(column_schema, std::move(record_reader), !is_nested, reader); -} - -// 1. RowGroupReader::GetColumnPageReader(leaf_column_id) -> Arrow PageReader -Status ParquetColumnReaderFactory::get_record_reader( - int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, const std::string& name, - bool install_page_filter, bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (_row_group == nullptr) { - return Status::InternalError("Parquet row group reader is not initialized for column {}", - name); - } - if (leaf_column_id < 0 || leaf_column_id >= static_cast(_record_readers.size())) { - return Status::InvalidArgument("Invalid parquet leaf column id {} for column {}", - leaf_column_id, name); - } - if (descriptor == nullptr) { - return Status::InvalidArgument("Parquet column descriptor is null for column {}", name); - } - auto& record_readers = read_dictionary ? _dictionary_record_readers : _record_readers; - if (record_readers[leaf_column_id] == nullptr) { - try { - auto page_reader = _row_group->GetColumnPageReader(leaf_column_id); - if (install_page_filter) { - install_data_page_filter(page_reader, _page_skip_plans, leaf_column_id, - _page_skip_profile); - } - const auto level_info = ::parquet::internal::LevelInfo::ComputeLevelInfo(descriptor); - record_readers[leaf_column_id] = ::parquet::internal::RecordReader::Make( - descriptor, level_info, ::arrow::default_memory_pool(), - /*read_dictionary=*/read_dictionary, - /*read_dense_for_nullable=*/false); - record_readers[leaf_column_id]->SetPageReader(std::move(page_reader)); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to create parquet record reader for column {}: {}", - name, e.what()); - } - } - if (record_readers[leaf_column_id] == nullptr) { - return Status::Corruption("Failed to create parquet record reader for column {}", name); - } - *reader = record_readers[leaf_column_id]; - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_struct_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - std::vector> child_readers; - child_readers.reserve(column_schema.children.size()); - std::vector child_output_indices; - child_output_indices.reserve(column_schema.children.size()); - DataTypes projected_child_types; - Strings projected_child_names; - for (size_t child_idx = 0; child_idx < column_schema.children.size(); ++child_idx) { - const auto& child_schema = column_schema.children[child_idx]; - const auto* child_projection = - format::find_child_projection(projection, child_schema->local_id); - if (!format::is_child_projected(projection, child_schema->local_id)) { - continue; - } - std::unique_ptr child_reader; - RETURN_IF_ERROR( - create_column_reader(*child_schema, child_projection, true, false, &child_reader)); - child_output_indices.push_back(static_cast(projected_child_types.size())); - projected_child_types.push_back(make_nullable(child_reader->type())); - projected_child_names.push_back(child_reader->name()); - child_readers.push_back(std::move(child_reader)); - } - if (format::is_partial_projection(projection) && - projected_child_types.size() != projection->children.size()) { - return Status::InvalidArgument( - "Parquet STRUCT projection for column {} contains invalid child", - column_schema.name); - } - if (projected_child_types.empty() && !column_schema.children.empty()) { - return Status::NotSupported("Parquet STRUCT projection for column {} contains no children", - column_schema.name); - } - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(projection)) { - type = std::make_shared(projected_child_types, projected_child_names); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique( - column_schema, std::move(type), std::move(child_readers), - std::move(child_output_indices), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_list_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for column {}", - column_schema.name); - } - std::unique_ptr element_reader; - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - if (format::is_partial_projection(projection) && element_projection == nullptr) { - return Status::NotSupported("Parquet LIST projection for column {} contains no element", - column_schema.name); - } - RETURN_IF_ERROR( - create_column_reader(element_schema, element_projection, true, false, &element_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(element_projection)) { - type = std::make_shared(element_reader->type()); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = std::make_unique(column_schema, std::move(type), - std::move(element_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create_map_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - if (column_schema.children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP layout for column {}", - column_schema.name); - } - const auto& key_schema = *column_schema.children[0]; - const auto& value_schema = *column_schema.children[1]; - const auto* value_projection = format::find_child_projection(projection, value_schema.local_id); - if (format::is_partial_projection(projection)) { - if (value_projection == nullptr) { - return Status::NotSupported("Parquet MAP projection for column {} contains no value", - column_schema.name); - } - for (const auto& child_projection : projection->children) { - if (child_projection.local_id() == key_schema.local_id) { - continue; - } - if (child_projection.local_id() != value_schema.local_id) { - return Status::InvalidArgument( - "Parquet MAP projection for column {} contains invalid child", - column_schema.name); - } - } - } - std::unique_ptr key_reader; - // MAP materialization always needs the full key stream. It owns entry existence, offsets and - // key equality semantics, so MAP projection is defined only as value-subtree pruning. - RETURN_IF_ERROR(create_column_reader(key_schema, nullptr, true, false, &key_reader)); - std::unique_ptr value_reader; - RETURN_IF_ERROR( - create_column_reader(value_schema, value_projection, true, false, &value_reader)); - DataTypePtr type = column_schema.type; - if (format::is_partial_projection(value_projection)) { - type = std::make_shared(make_nullable(key_reader->type()), - make_nullable(value_reader->type())); - if (column_schema.type != nullptr && column_schema.type->is_nullable()) { - type = make_nullable(type); - } - } - *reader = - std::make_unique(column_schema, std::move(type), std::move(key_reader), - std::move(value_reader), _column_reader_profile); - return Status::OK(); -} - -Status ParquetColumnReaderFactory::create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, - bool read_dictionary) const { - return create_column_reader(column_schema, projection, false, read_dictionary, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const { - return create_count_shape_reader_impl(column_schema, projection, false, reader); -} - -Status ParquetColumnReaderFactory::create_count_shape_reader_impl( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, is_nested, false, reader); - case ParquetColumnSchemaKind::STRUCT: { - if (column_schema.children.empty()) { - return Status::NotSupported("Parquet COUNT shape reader found empty STRUCT column {}", - column_schema.name); - } - const ParquetColumnSchema* child_schema = nullptr; - const format::LocalColumnIndex* child_projection = nullptr; - if (format::is_partial_projection(projection)) { - const auto child_id = projection->children[0].local_id(); - const auto child_it = std::ranges::find_if( - column_schema.children, - [&](const auto& child) { return child->local_id == child_id; }); - if (child_it == column_schema.children.end()) { - return Status::InvalidArgument( - "Parquet COUNT projection for column {} contains invalid child", - column_schema.name); - } - child_schema = child_it->get(); - child_projection = &projection->children[0]; - } else { - child_schema = column_schema.children[0].get(); - } - DORIS_CHECK(child_schema != nullptr); - return create_count_shape_reader_impl(*child_schema, child_projection, true, reader); - } - case ParquetColumnSchemaKind::LIST: { - if (column_schema.children.size() != 1) { - return Status::NotSupported("Unsupported parquet LIST layout for COUNT column {}", - column_schema.name); - } - const auto& element_schema = *column_schema.children[0]; - const auto* element_projection = - format::find_child_projection(projection, element_schema.local_id); - return create_count_shape_reader_impl(element_schema, element_projection, true, reader); - } - case ParquetColumnSchemaKind::MAP: { - if (column_schema.children.empty()) { - return Status::NotSupported("Unsupported parquet MAP layout for COUNT column {}", - column_schema.name); - } - // The key stream defines MAP entry existence and offsets. Counting top-level MAP NULL-ness - // from it avoids creating a value reader, which is the expensive path for files with huge - // MAP value strings. - return create_count_shape_reader_impl(*column_schema.children[0], nullptr, true, reader); - } - } - return Status::NotSupported("Unsupported parquet column schema kind for COUNT column {}", - column_schema.name); -} - -Status ParquetColumnReaderFactory::create_column_reader( - const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - bool is_nested, bool read_dictionary, std::unique_ptr* reader) const { - if (reader == nullptr) { - return Status::InvalidArgument("reader is null"); - } - switch (column_schema.kind) { - case ParquetColumnSchemaKind::PRIMITIVE: - if (is_nested) { - if (format::is_partial_projection(projection)) { - return Status::InvalidArgument("Parquet scalar projection is invalid for column {}", - column_schema.name); - } - return create_scalar_column_reader(column_schema, true, false, reader); - } - return create_scalar_column_reader(column_schema, false, read_dictionary, reader); - case ParquetColumnSchemaKind::STRUCT: - return create_struct_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::LIST: - return create_list_column_reader(column_schema, projection, reader); - case ParquetColumnSchemaKind::MAP: - return create_map_column_reader(column_schema, projection, reader); - } - return Status::NotSupported("Unsupported parquet column schema kind for column {}", - column_schema.name); -} - -ParquetColumnReader::ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, - ParquetColumnReaderProfile profile) - : _profile(profile), - _field_id(schema.local_id), - _leaf_column_id(schema.leaf_column_id), - _nullable_definition_level(schema.nullable_definition_level), - _repeated_repetition_level(schema.repeated_repetition_level), - _definition_level(schema.definition_level), - _repetition_level(schema.repetition_level), - _repeated_ancestor_definition_level(schema.repeated_ancestor_definition_level), - _type(std::move(type)), - _name(schema.name) {} - -Status ParquetColumnReader::load_nested_batch(int64_t) { - return Status::NotSupported("Parquet nested batch load is not supported for column {}", _name); -} - -Status ParquetColumnReader::load_nested_levels_batch(int64_t) { - return Status::NotSupported("Parquet nested levels batch load is not supported for column {}", - _name); -} - -Status ParquetColumnReader::build_nested_column(int64_t, MutableColumnPtr&, int64_t*) { - return Status::NotSupported("Parquet nested column build is not supported for column {}", - _name); -} - -Status ParquetColumnReader::consume_nested_column(int64_t, int64_t*) { - return Status::NotSupported("Parquet nested column consume is not supported for column {}", - _name); -} - -Status ParquetColumnReader::skip_nested_rows(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - // A nested parent row may expand to many child values. Capping the number of parent rows per - // loaded batch bounds that amplification for large holes. The consume interface advances the - // loaded definition/repetition levels recursively without constructing a discarded Column. - constexpr int64_t MAX_NESTED_SKIP_BATCH_SIZE = 4096; - int64_t remaining_rows = rows; - while (remaining_rows > 0) { - const int64_t batch_rows = std::min(remaining_rows, MAX_NESTED_SKIP_BATCH_SIZE); - RETURN_IF_ERROR(load_nested_levels_batch(batch_rows)); - int64_t rows_consumed = 0; - RETURN_IF_ERROR(consume_nested_column(batch_rows, &rows_consumed)); - if (rows_consumed != batch_rows) { - return Status::Corruption( - "Failed to skip nested parquet column {}: skipped {} of {} rows in batch", - _name, rows_consumed, batch_rows); - } - remaining_rows -= batch_rows; - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -const std::vector& ParquetColumnReader::nested_definition_levels() const { - static const std::vector empty; - return empty; -} - -const std::vector& ParquetColumnReader::nested_repetition_levels() const { - static const std::vector empty; - return empty; -} - -int64_t ParquetColumnReader::nested_levels_written() const { - return 0; -} - -bool ParquetColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 51dbd44c11c226..6ea0e62960fb5e 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -16,216 +16,63 @@ #pragma once #include -#include -#include #include -#include #include "common/status.h" -#include "core/column/column_nullable.h" +#include "core/column/column.h" #include "core/data_type/data_type.h" -#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" #include "format_v2/parquet/selection_vector.h" -#include "runtime/runtime_profile.h" - -namespace parquet { -class ColumnDescriptor; -class RowGroupReader; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris { -class IColumn; -} // namespace doris namespace doris::format::parquet { struct ParquetColumnSchema; +// Scan-time column contract for FileScannerV2. +// +// Physical Parquet columns are implemented by NativeColumnReader, which owns Doris' native page +// decoder and writes directly into the destination Doris column. Synthetic row-position/global-id +// readers implement the same cursor contract. Arrow RecordReader, ParquetLeafBatch, decoded value +// views, and the former nested build/consume protocol intentionally do not belong to this API. class ParquetColumnReader { public: virtual ~ParquetColumnReader() = default; virtual int file_column_id() const { return _field_id; } - virtual int parquet_leaf_column_id() const { return _leaf_column_id; } - - int16_t nullable_definition_level() const { return _nullable_definition_level; } - int16_t repeated_repetition_level() const { return _repeated_repetition_level; } - virtual const DataTypePtr& type() const { return _type; } virtual const std::string& name() const { return _name; } const ParquetColumnReaderProfile& profile() const { return _profile; } + // Consume rows from the row-group cursor and append them to column. virtual Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) = 0; + // Consume rows without materializing values. virtual Status skip(int64_t rows); - virtual Status select(const SelectionVector& sel, uint16_t selected_rows, int64_t batch_rows, - MutableColumnPtr& column); + // Consume batch_rows and append only selection[0, selected_rows). The default implementation + // coalesces adjacent indices into ranges; native readers override it with one decoder call. + virtual Status select(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, MutableColumnPtr& column); - virtual Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, bool* used_filter); - - virtual Status load_nested_batch(int64_t rows); - - // Shape-only load interface for COUNT(col) and skip. Implementations guarantee only that - // nested_definition_levels(), nested_repetition_levels(), and nested_levels_written() are - // available; value indices and payload columns may be absent. Callers may inspect the levels or - // call consume_nested_column(), but must not call build_nested_column() afterwards. For example, - // skipping ARRAY uses this method to find ARRAY boundaries without constructing a - // ColumnString. The underlying Arrow reader may still decode a page because it has no public - // levels-only API. Normal scans that need output values use load_nested_batch() instead. - virtual Status load_nested_levels_batch(int64_t rows); - - virtual Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read); - - // Consume logical values from a batch previously loaded by load_nested_batch() or - // load_nested_levels_batch() without appending them to an output Column. Implementations must - // advance exactly the same nested level cursors and perform the same shape/null/alignment - // validation as build_nested_column(). The levels-only form is preferred for skip paths because - // it avoids transferring leaf payloads into Doris Columns when they will be discarded. - // - // `length_upper_bound` is expressed at this reader's logical level, not in physical leaf - // values. For example, consuming two rows from ARRAY [[1, 2], []] consumes two parent ARRAY - // rows but only two element values. A MAP implementation must also consume key/value streams - // in lockstep, while a nullable STRUCT consumes no child value for a null parent. - // - // Callers must not use the ordinary skip() after either load call: the leaf stream has already - // advanced into an in-memory nested batch, and doing so would advance it twice. - // `values_consumed` may be smaller than the requested bound only when the loaded batch ends. - virtual Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed); - - virtual const std::vector& nested_definition_levels() const; - virtual const std::vector& nested_repetition_levels() const; - virtual int64_t nested_levels_written() const; - virtual bool is_or_has_repeated_child() const; - virtual void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level); - - int64_t nested_build_level_cursor() const { return _nested_build_level_cursor; } - void set_nested_build_level_cursor(int64_t cursor) { - DORIS_CHECK(cursor >= 0); - _nested_build_level_cursor = cursor; - } - void reset_nested_build_level_cursor() { _nested_build_level_cursor = 0; } + virtual Status select_with_dictionary_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, bool* used_filter); protected: - ParquetColumnReader(const ParquetColumnSchema& schema, const DataTypePtr type, + ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, ParquetColumnReaderProfile profile = {}); ParquetColumnReader() = default; - // Load shape levels and consume skipped parent rows in bounded batches. The bound limits level - // memory when a parent expands to many children; the levels-only load plus - // consume_nested_column() avoids payload materialization and output Columns. - Status skip_nested_rows(int64_t rows); + void update_reader_read_rows(int64_t rows) const; void update_reader_skip_rows(int64_t rows) const; ParquetColumnReaderProfile _profile; - const int _field_id = -1; // child ordinal in the parent node - const int _leaf_column_id = -1; // Parquet physical leaf column id (-1 = non-leaf) - const int16_t _nullable_definition_level = - 0; // definition-level threshold where this node becomes nullable - const int16_t _repeated_repetition_level = - 0; // repetition level of the nearest repeated ancestor - const int16_t _definition_level = 0; // definition level accumulated to this node - const int16_t _repetition_level = 0; // repetition level accumulated to this node - const int16_t _repeated_ancestor_definition_level = - 0; // definition level of the nearest repeated ancestor - const DataTypePtr _type; // Doris target type - const std::string _name; // column name for error messages - int64_t _nested_build_level_cursor = 0; // nested build cursor (current level position) + const int _field_id = -1; + const int _leaf_column_id = -1; + const DataTypePtr _type; + const std::string _name; }; -class ParquetColumnReaderFactory { -public: - ParquetColumnReaderFactory(std::shared_ptr<::parquet::RowGroupReader> row_group, - int num_leaf_columns, - const std::map* page_skip_plans = nullptr, - ParquetPageSkipProfile page_skip_profile = {}, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false, - ParquetColumnReaderProfile column_reader_profile = {}); - - Status create(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader, bool read_dictionary = false) const; - - // Create a scalar reader for one representative leaf that carries the top-level column shape. - // This is used by COUNT(col): the caller needs definition/repetition levels to decide whether - // the top-level value is NULL, but must not materialize heavy payload leaves. MAP deliberately - // uses the key leaf because the key stream owns entry existence and avoids reading value pages. - Status create_count_shape_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create(const ParquetColumnSchema& column_schema, - std::unique_ptr* reader) const { - return create(column_schema, nullptr, reader); - } - - std::unique_ptr create_row_position_column_reader( - int64_t row_group_first_row) const; - std::unique_ptr create_global_rowid_column_reader( - const format::GlobalRowIdContext& context, int64_t row_group_first_row) const; - -private: - Status create_scalar_column_reader(const ParquetColumnSchema& column_schema, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - - Status create_struct_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_list_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_map_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::unique_ptr* reader) const; - - Status create_column_reader(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, bool is_nested, - bool read_dictionary, - std::unique_ptr* reader) const; - Status create_count_shape_reader_impl(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - bool is_nested, - std::unique_ptr* reader) const; - - Status get_record_reader(int leaf_column_id, const ::parquet::ColumnDescriptor* descriptor, - const std::string& name, bool install_page_filter, - bool read_dictionary, - std::shared_ptr<::parquet::internal::RecordReader>* reader) const; - - Status make_scalar_column_reader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - bool use_page_skip_plan, std::unique_ptr* reader) const; - - std::shared_ptr<::parquet::RowGroupReader> _row_group; // Arrow RowGroup reader - mutable std::vector> - _record_readers; // RecordReader cache by leaf_column_id - mutable std::vector> - _dictionary_record_readers; // dictionary-exposing RecordReader cache by leaf_column_id - const std::map* _page_skip_plans = - nullptr; // page-index pruning result - ParquetPageSkipProfile _page_skip_profile; // page skip profile - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - ParquetColumnReaderProfile _column_reader_profile; // column reader profile -}; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp new file mode 100644 index 00000000000000..30fd395359574e --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/count_column_reader.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "format_v2/parquet/parquet_column_schema.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet { +namespace { + +Status find_count_leaf(const ParquetColumnSchema& schema, + const format::LocalColumnIndex* projection, + const ParquetColumnSchema** leaf) { + switch (schema.kind) { + case ParquetColumnSchemaKind::PRIMITIVE: + if (format::is_partial_projection(projection)) { + return Status::InvalidArgument("Parquet COUNT projection is invalid for column {}", + schema.name); + } + *leaf = &schema; + return Status::OK(); + case ParquetColumnSchemaKind::STRUCT: { + DORIS_CHECK(!schema.children.empty()); + if (!format::is_partial_projection(projection)) { + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + const auto child_id = projection->children.front().local_id(); + const auto child = std::ranges::find_if( + schema.children, + [child_id](const auto& candidate) { return candidate->local_id == child_id; }); + if (child == schema.children.end()) { + return Status::InvalidArgument( + "Parquet COUNT projection for column {} contains invalid child", schema.name); + } + return find_count_leaf(**child, &projection->children.front(), leaf); + } + case ParquetColumnSchemaKind::LIST: { + DORIS_CHECK(schema.children.size() == 1); + const auto& element = *schema.children.front(); + return find_count_leaf(element, + format::find_child_projection(projection, element.local_id), leaf); + } + case ParquetColumnSchemaKind::MAP: + // The key stream defines entry existence and top-level MAP shape. Never select the value + // leaf: a COUNT(map_col) must not retain huge value strings merely to inspect nullability. + DORIS_CHECK(!schema.children.empty()); + return find_count_leaf(*schema.children.front(), nullptr, leaf); + } + return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); +} + +bool is_binary_physical_type(const ::parquet::ColumnDescriptor& descriptor) { + return descriptor.physical_type() == ::parquet::Type::BYTE_ARRAY || + descriptor.physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY; +} + +} // namespace + +CountColumnReader::CountColumnReader( + const ParquetColumnSchema& leaf_schema, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + ParquetColumnReaderProfile profile) + : _leaf_schema(leaf_schema), + _record_reader(std::move(record_reader)), + _profile(profile), + _name(leaf_schema.name) {} + +Status CountColumnReader::create(std::shared_ptr<::parquet::RowGroupReader> row_group, + const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader) { + DORIS_CHECK(row_group != nullptr); + DORIS_CHECK(reader != nullptr); + const ParquetColumnSchema* leaf = nullptr; + RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf)); + DORIS_CHECK(leaf != nullptr); + DORIS_CHECK(leaf->leaf_column_id >= 0); + DORIS_CHECK(leaf->descriptor != nullptr); + + try { + auto page_reader = row_group->GetColumnPageReader(leaf->leaf_column_id); + DORIS_CHECK(page_reader != nullptr); + const auto level_info = + ::parquet::internal::LevelInfo::ComputeLevelInfo(leaf->descriptor); + auto record_reader = ::parquet::internal::RecordReader::Make( + leaf->descriptor, level_info, ::arrow::default_memory_pool(), + /*read_dictionary=*/false, + /*read_dense_for_nullable=*/false); + DORIS_CHECK(record_reader != nullptr); + record_reader->SetPageReader(std::move(page_reader)); + reader->reset(new CountColumnReader(*leaf, std::move(record_reader), profile)); + return Status::OK(); + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to create Parquet COUNT reader for column {}: {}", + leaf->name, e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to create Parquet COUNT reader for column {}: {}", + leaf->name, e.what()); + } +} + +Status CountColumnReader::skip(int64_t rows) { + DORIS_CHECK(rows >= 0); + if (rows == 0) { + return Status::OK(); + } + int64_t skipped_rows = 0; + try { + _record_reader->Reset(); + SCOPED_TIMER(_profile.arrow_skip_records_time); + while (skipped_rows < rows) { + const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); + if (skipped <= 0) { + return Status::Corruption( + "Parquet COUNT reader skipped {} of {} rows for column {}", skipped_rows, + rows, _name); + } + skipped_rows += skipped; + } + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to skip Parquet COUNT rows for column {}: {}", _name, + e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to skip Parquet COUNT rows for column {}: {}", _name, + e.what()); + } + if (_profile.reader_skip_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_skip_rows, rows); + } + return Status::OK(); +} + +Status CountColumnReader::release_binary_builder() { + DORIS_CHECK(_leaf_schema.descriptor != nullptr); + if (!is_binary_physical_type(*_leaf_schema.descriptor)) { + return Status::OK(); + } + auto* binary_reader = + dynamic_cast<::parquet::internal::BinaryRecordReader*>(_record_reader.get()); + if (binary_reader == nullptr) { + return Status::InternalError( + "Parquet COUNT binary reader is unavailable for column {}", _name); + } + // GetBuilderChunks transfers builder ownership. Keep the result local so binary payload pages + // are released before the next batch and never overlap adaptive-batch allocations. + auto discarded_chunks = binary_reader->GetBuilderChunks(); + discarded_chunks.clear(); + return Status::OK(); +} + +Status CountColumnReader::read_levels(int64_t rows, int64_t* rows_read) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(rows_read != nullptr); + _definition_levels.clear(); + _repetition_levels.clear(); + _levels_written = 0; + if (rows == 0) { + *rows_read = 0; + return Status::OK(); + } + + try { + _record_reader->Reset(); + _record_reader->Reserve(rows); + { + SCOPED_TIMER(_profile.arrow_read_records_time); + *rows_read = _record_reader->ReadRecords(rows); + } + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to read Parquet COUNT levels for column {}: {}", _name, + e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to read Parquet COUNT levels for column {}: {}", _name, + e.what()); + } + if (*rows_read < 0 || *rows_read > rows) { + return Status::Corruption("Invalid Parquet COUNT row count {} for column {}", *rows_read, + _name); + } + + _levels_written = _record_reader->levels_position(); + if (_levels_written > _record_reader->levels_written()) { + return Status::Corruption( + "Invalid Parquet COUNT level position {} of {} for column {}", _levels_written, + _record_reader->levels_written(), _name); + } + const auto* descriptor = _leaf_schema.descriptor; + if (_levels_written == 0 && *rows_read > 0 && descriptor->max_definition_level() == 0 && + descriptor->max_repetition_level() == 0) { + _levels_written = *rows_read; + } + if (_levels_written < *rows_read) { + return Status::Corruption( + "Parquet COUNT returned {} levels for {} rows in column {}", _levels_written, + *rows_read, _name); + } + + _definition_levels.resize(static_cast(_levels_written)); + if (descriptor->max_definition_level() == 0) { + std::fill(_definition_levels.begin(), _definition_levels.end(), 0); + } else { + const auto* levels = _record_reader->def_levels(); + DORIS_CHECK(levels != nullptr || _levels_written == 0); + std::copy_n(levels, _levels_written, _definition_levels.begin()); + } + _repetition_levels.resize(static_cast(_levels_written)); + if (descriptor->max_repetition_level() == 0) { + std::fill(_repetition_levels.begin(), _repetition_levels.end(), 0); + } else { + const auto* levels = _record_reader->rep_levels(); + DORIS_CHECK(levels != nullptr || _levels_written == 0); + std::copy_n(levels, _levels_written, _repetition_levels.begin()); + } + RETURN_IF_ERROR(release_binary_builder()); + if (_profile.reader_read_rows != nullptr) { + COUNTER_UPDATE(_profile.reader_read_rows, *rows_read); + } + return Status::OK(); +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h new file mode 100644 index 00000000000000..a02c2988ed0eae --- /dev/null +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/column_data.h" +#include "format_v2/parquet/parquet_profile.h" + +namespace parquet { +class RowGroupReader; +namespace internal { +class RecordReader; +} // namespace internal +} // namespace parquet + +namespace doris::format::parquet { +struct ParquetColumnSchema; + +// Isolated compatibility reader for the existing COUNT(nullable_col) pushdown. +// +// Ordinary scans never instantiate this class. COUNT needs only Dremel definition/repetition +// levels, so this reader exposes exactly one shape operation and no value-materialization API. +// Arrow currently has no public levels-only page decoder; ReadRecords therefore advances its +// private RecordReader, after which binary builder chunks are immediately released and only the +// copied level vectors survive. Keeping this exception isolated prevents Arrow arrays/builders, +// ParquetLeafBatch, and decoded-value views from leaking back into the scan reader contract. +class CountColumnReader { +public: + static Status create(std::shared_ptr<::parquet::RowGroupReader> row_group, + const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, + ParquetColumnReaderProfile profile, + std::unique_ptr* reader); + + Status skip(int64_t rows); + Status read_levels(int64_t rows, int64_t* rows_read); + + const std::vector& definition_levels() const { return _definition_levels; } + const std::vector& repetition_levels() const { return _repetition_levels; } + int64_t levels_written() const { return _levels_written; } + +private: + CountColumnReader(const ParquetColumnSchema& leaf_schema, + std::shared_ptr<::parquet::internal::RecordReader> record_reader, + ParquetColumnReaderProfile profile); + + Status release_binary_builder(); + + const ParquetColumnSchema& _leaf_schema; + std::shared_ptr<::parquet::internal::RecordReader> _record_reader; + ParquetColumnReaderProfile _profile; + std::string _name; + std::vector _definition_levels; + std::vector _repetition_levels; + int64_t _levels_written = 0; +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.cpp b/be/src/format_v2/parquet/reader/list_column_reader.cpp deleted file mode 100644 index 5fc08f8fae31cf..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.cpp +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/list_column_reader.h" - -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status ListColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status ListColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status ListColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_batch(rows); -} - -Status ListColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_element_reader != nullptr); - reset_nested_build_level_cursor(); - return _element_reader->load_nested_levels_batch(rows); -} - -Status ListColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet list build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status ListColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status ListColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet list process result pointer for column {}", - _name); - } - DORIS_CHECK(_element_reader != nullptr); - ColumnArray* array_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr nested_column; - if (column != nullptr) { - array_column = array_column_from_output(*column); - DORIS_CHECK(array_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - nested_column = array_column->get_data_ptr()->assert_mutable(); - const auto& element_output_type = - assert_cast(*remove_nullable(_type)).get_nested_type(); - remove_nullable_wrapper_if_not_expected(element_output_type, &nested_column); - } - - const auto& def_levels = _element_reader->nested_definition_levels(); - const auto& rep_levels = _element_reader->nested_repetition_levels(); - const int64_t levels_written = _element_reader->nested_levels_written(); - _entry_counts.clear(); - _parent_nulls.clear(); - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - if (rep_level == _repetition_level) { - if (_entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet LIST column {}", - _name); - } - if (def_level >= _definition_level) { - ++_entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet LIST column {} contains null for non-nullable LIST", - _name); - } - _parent_nulls.push_back(parent_is_null); - _entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - int64_t child_value_count = 0; - if (!_element_reader->is_or_has_repeated_child()) { - for (const auto entry_count : _entry_counts) { - total_entries += entry_count; - } - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(total_entries), nested_column, &child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(total_entries), &child_value_count)); - } - } else { - uint64_t pending_entries = 0; - auto flush_pending_entries = [&]() -> Status { - if (pending_entries == 0) { - return Status::OK(); - } - int64_t span_child_value_count = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_element_reader->build_nested_column( - static_cast(pending_entries), nested_column, - &span_child_value_count)); - } else { - RETURN_IF_ERROR(_element_reader->consume_nested_column( - static_cast(pending_entries), &span_child_value_count)); - } - if (span_child_value_count != static_cast(pending_entries)) { - return Status::Corruption( - "Parquet LIST column {} built {} child values, expected {}", _name, - span_child_value_count, pending_entries); - } - child_value_count += span_child_value_count; - pending_entries = 0; - return Status::OK(); - }; - - for (const auto entry_count : _entry_counts) { - total_entries += entry_count; - if (entry_count > 0) { - pending_entries += entry_count; - continue; - } - RETURN_IF_ERROR(flush_pending_entries()); - _element_reader->advance_nested_build_level_cursor_past_parent(_repetition_level); - } - RETURN_IF_ERROR(flush_pending_entries()); - } - if (child_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet LIST column {} built {} child values, expected {}", - _name, child_value_count, total_entries); - } - if (column != nullptr) { - array_column->get_data_ptr() = std::move(nested_column); - append_offsets(array_column->get_offsets(), _entry_counts); - append_parent_nulls(parent_null_map, _parent_nulls); - } - return Status::OK(); -} - -const std::vector& ListColumnReader::nested_definition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_definition_levels(); -} - -const std::vector& ListColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_repetition_levels(); -} - -int64_t ListColumnReader::nested_levels_written() const { - DORIS_CHECK(_element_reader != nullptr); - return _element_reader->nested_levels_written(); -} - -bool ListColumnReader::is_or_has_repeated_child() const { - return true; -} - -void ListColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_element_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _element_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/list_column_reader.h b/be/src/format_v2/parquet/reader/list_column_reader.h deleted file mode 100644 index ac3d44651846d8..00000000000000 --- a/be/src/format_v2/parquet/reader/list_column_reader.h +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class ListColumnReader final : public ParquetColumnReader { -public: - ListColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr element_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _element_reader(std::move(element_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr - _element_reader; // element reader (recursive; may be Scalar/Struct/List/Map) - // A list row such as [[1, 2], NULL, []] becomes entry counts [2, 0, 0] and parent nulls - // [0, 1, 0]. These vectors are batch scratch, not reader state; clear() keeps their capacity. - std::vector _entry_counts; - NullMap _parent_nulls; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.cpp b/be/src/format_v2/parquet/reader/map_column_reader.cpp deleted file mode 100644 index 0d165b01e2fb52..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.cpp +++ /dev/null @@ -1,285 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/map_column_reader.h" - -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { -namespace { - -void remove_nullable_wrapper_if_not_expected(const DataTypePtr& output_type, - MutableColumnPtr* column) { - DORIS_CHECK(column != nullptr); - if (output_type->is_nullable()) { - return; - } - if (auto* nullable_column = check_and_get_column(**column)) { - *column = nullable_column->get_nested_column_ptr(); - } -} - -} // namespace - -Status MapColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status MapColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status MapColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_batch(rows)); - return _value_reader->load_nested_batch(rows); -} - -Status MapColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(_key_reader->load_nested_levels_batch(rows)); - return _value_reader->load_nested_levels_batch(rows); -} - -Status MapColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet map build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status MapColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status MapColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument("Invalid parquet map process result pointer for column {}", - _name); - } - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ColumnMap* map_column = nullptr; - NullMap* parent_null_map = nullptr; - MutableColumnPtr key_column; - MutableColumnPtr value_column; - if (column != nullptr) { - map_column = map_column_from_output(*column); - DORIS_CHECK(map_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - key_column = map_column->get_keys_ptr()->assert_mutable(); - value_column = map_column->get_values_ptr()->assert_mutable(); - const auto& map_output_type = assert_cast(*remove_nullable(_type)); - remove_nullable_wrapper_if_not_expected(map_output_type.get_key_type(), &key_column); - remove_nullable_wrapper_if_not_expected(map_output_type.get_value_type(), &value_column); - } - - const auto& def_levels = _key_reader->nested_definition_levels(); - const auto& rep_levels = _key_reader->nested_repetition_levels(); - const int64_t levels_written = _key_reader->nested_levels_written(); - - _entry_counts.clear(); - _map_level_indices.clear(); - _parent_nulls.clear(); - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - const int16_t min_parent_definition_level = - static_cast(_definition_level - 1 - (_type->is_nullable() ? 1 : 0)); - while (level_idx < levels_written) { - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = rep_level < _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - const int64_t current_level_idx = level_idx; - ++level_idx; - if (rep_level > _repetition_level || def_level < min_parent_definition_level || - (!starts_parent && def_level < _repeated_ancestor_definition_level)) { - continue; - } - _map_level_indices.push_back(current_level_idx); - if (rep_level == _repetition_level) { - if (_entry_counts.empty()) { - return Status::Corruption("Invalid repeated level for parquet MAP column {}", - _name); - } - if (def_level >= _definition_level) { - ++_entry_counts.back(); - } - continue; - } - - const bool parent_is_null = def_level < _definition_level - 1; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable MAP", - _name); - } - _parent_nulls.push_back(parent_is_null); - _entry_counts.push_back(def_level >= _definition_level ? 1 : 0); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - uint64_t total_entries = 0; - for (const auto entry_count : _entry_counts) { - total_entries += entry_count; - } - int64_t key_value_count = 0; - size_t key_start = 0; - if (column != nullptr) { - key_start = key_column->size(); - RETURN_IF_ERROR(_key_reader->build_nested_column(static_cast(total_entries), - key_column, &key_value_count)); - } else if (auto* scalar_key_reader = dynamic_cast(_key_reader.get())) { - // MAP keys are required even if a projected Doris key type is nullable. Validate each - // actual entry directly from the key level stream while advancing past empty/null maps. - for (const int64_t key_level_idx : _map_level_indices) { - if (def_levels[key_level_idx] >= _definition_level) { - RETURN_IF_ERROR(scalar_key_reader->validate_nested_value(key_level_idx, true)); - ++key_value_count; - } - } - scalar_key_reader->set_nested_build_level_cursor(level_idx); - } else { - RETURN_IF_ERROR(_key_reader->consume_nested_column(static_cast(total_entries), - &key_value_count)); - } - if (key_value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} keys, expected {}", _name, - key_value_count, total_entries); - } - if (column != nullptr) { - if (const auto* nullable_key_column = check_and_get_column(*key_column); - nullable_key_column != nullptr && - nullable_key_column->has_null(key_start, nullable_key_column->size())) { - return Status::Corruption("Parquet MAP column {} contains null key", _name); - } - } - int64_t value_count = 0; - if (auto* scalar_value_reader = dynamic_cast(_value_reader.get())) { - const auto& value_def_levels = scalar_value_reader->nested_definition_levels(); - const auto& value_rep_levels = scalar_value_reader->nested_repetition_levels(); - const int64_t value_levels_written = scalar_value_reader->nested_levels_written(); - int64_t value_level_idx = scalar_value_reader->nested_build_level_cursor(); - for (const int64_t key_level_idx : _map_level_indices) { - while (value_level_idx < value_levels_written && - (value_rep_levels[value_level_idx] > _repetition_level || - value_def_levels[value_level_idx] < min_parent_definition_level || - (value_rep_levels[value_level_idx] >= _repetition_level && - value_def_levels[value_level_idx] < _repeated_ancestor_definition_level))) { - ++value_level_idx; - } - if (value_level_idx >= value_levels_written) { - return Status::Corruption( - "Parquet MAP column {} value stream ended before key stream", _name); - } - // MAP is encoded as a repeated key/value struct. The key stream owns entry existence, - // but the value stream still has one shape slot for every consumed MAP slot. Consume - // value slots in lockstep with key slots so shape-only slots from empty/null maps do - // not become scalar values. - if (value_rep_levels[value_level_idx] != rep_levels[key_level_idx]) { - return Status::Corruption( - "Parquet MAP column {} value repetition level is not aligned with key " - "stream", - _name); - } - if (def_levels[key_level_idx] >= _definition_level) { - if (column != nullptr) { - RETURN_IF_ERROR(scalar_value_reader->append_nested_value(value_level_idx, - value_column)); - } else { - RETURN_IF_ERROR( - scalar_value_reader->validate_nested_value(value_level_idx, false)); - } - ++value_count; - } - ++value_level_idx; - } - scalar_value_reader->set_nested_build_level_cursor(value_level_idx); - } else { - // Complex MAP values own their nested shape below the entry slot, so they recursively - // process exactly one child value for each MAP entry. - if (column != nullptr) { - RETURN_IF_ERROR(_value_reader->build_nested_column(static_cast(total_entries), - value_column, &value_count)); - } else { - RETURN_IF_ERROR(_value_reader->consume_nested_column( - static_cast(total_entries), &value_count)); - } - } - if (value_count != static_cast(total_entries)) { - return Status::Corruption("Parquet MAP column {} built {} values, expected {}", _name, - value_count, total_entries); - } - - if (column != nullptr) { - map_column->get_keys_ptr() = std::move(key_column); - map_column->get_values_ptr() = std::move(value_column); - append_offsets(map_column->get_offsets(), _entry_counts); - append_parent_nulls(parent_null_map, _parent_nulls); - } - return Status::OK(); -} - -const std::vector& MapColumnReader::nested_definition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_definition_levels(); -} - -const std::vector& MapColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_repetition_levels(); -} - -int64_t MapColumnReader::nested_levels_written() const { - DORIS_CHECK(_key_reader != nullptr); - return _key_reader->nested_levels_written(); -} - -bool MapColumnReader::is_or_has_repeated_child() const { - return true; -} - -void MapColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - DORIS_CHECK(_key_reader != nullptr); - DORIS_CHECK(_value_reader != nullptr); - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _key_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - _value_reader->advance_nested_build_level_cursor_past_parent(parent_repetition_level); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/map_column_reader.h b/be/src/format_v2/parquet/reader/map_column_reader.h deleted file mode 100644 index df81b29137f152..00000000000000 --- a/be/src/format_v2/parquet/reader/map_column_reader.h +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -// 2. build_nested_column() -> -class MapColumnReader final : public ParquetColumnReader { -public: - MapColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::unique_ptr key_reader, - std::unique_ptr value_reader, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _key_reader(std::move(key_reader)), - _value_reader(std::move(value_reader)) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - - std::unique_ptr _key_reader; // key column reader (always read fully) - std::unique_ptr - _value_reader; // value column reader (can be pruned by projection) - // Key levels own MAP entry existence. The level index vector lets the value stream advance in - // lockstep without building another per-row lookup table; all three buffers retain capacity. - std::vector _entry_counts; - std::vector _map_level_indices; - NullMap _parent_nulls; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 92fc5681fa58ec..2f70d1acc8d96c 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -120,7 +120,7 @@ void collect_projected_ids(const ParquetColumnSchema& schema, DORIS_CHECK(!native_field.children.empty()); // MAP entry existence and offsets are owned by the key stream even for value-only // projections. Keep the key reader live and validate key/value entry alignment in v1's - // native MapColumnReader. + // native complex-column reader. ids->insert(native_field.children[0].get_column_id()); } } diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 8b66afac81d01c..783982d5c41d2a 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -29,6 +29,7 @@ #include "format/parquet/parquet_common.h" #include "format/parquet/vparquet_column_reader.h" #include "format/table/table_schema_change_helper.h" +#include "format_v2/column_data.h" #include "format_v2/parquet/reader/column_reader.h" namespace doris { diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp b/be/src/format_v2/parquet/reader/nested_column_materializer.cpp deleted file mode 100644 index e06b7eaaf317e7..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.cpp +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/nested_column_materializer.h" - -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" - -namespace doris::format::parquet { - -ColumnArray* array_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnMap* map_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return assert_cast(&nullable_column->get_nested_column()); - } - return assert_cast(column.get()); -} - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column) { - if (auto* nullable_column = check_and_get_column(*column)) { - return &nullable_column->get_null_map_data(); - } - return nullptr; -} - -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts) { - offsets.reserve(offsets.size() + entry_counts.size()); - uint64_t current_offset = offsets.empty() ? 0 : offsets.back(); - for (const auto entry_count : entry_counts) { - current_offset += entry_count; - offsets.push_back(current_offset); - } -} - -void append_parent_nulls(NullMap* dst, const NullMap& src) { - if (dst == nullptr) { - return; // target column is not nullable; no null marker is needed - } - dst->insert(src.begin(), src.end()); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/nested_column_materializer.h b/be/src/format_v2/parquet/reader/nested_column_materializer.h deleted file mode 100644 index 90fac01eb2f5e5..00000000000000 --- a/be/src/format_v2/parquet/reader/nested_column_materializer.h +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "core/column/column.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_struct.h" - -namespace doris::format::parquet { - -// ============================================================================ -// ============================================================================ - -ColumnArray* array_column_from_output(MutableColumnPtr& column); - -ColumnMap* map_column_from_output(MutableColumnPtr& column); - -ColumnStruct* struct_column_from_output(MutableColumnPtr& column); - -NullMap* null_map_from_nullable_output(MutableColumnPtr& column); - -// offsets[i] = offsets[i-1] + entry_counts[i]. -void append_offsets(ColumnArray::Offsets64& offsets, const std::vector& entry_counts); - -void append_parent_nulls(NullMap* dst, const NullMap& src); - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp b/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp deleted file mode 100644 index 08ecf5cb81d036..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.cpp +++ /dev/null @@ -1,850 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "core/string_ref.h" -#include "runtime/runtime_profile.h" -#include "util/defer_op.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -DecodedTimeUnit decoded_time_unit(ParquetTimeUnit time_unit) { - switch (time_unit) { - case ParquetTimeUnit::MILLIS: - return DecodedTimeUnit::MILLIS; - case ParquetTimeUnit::MICROS: - return DecodedTimeUnit::MICROS; - case ParquetTimeUnit::NANOS: - return DecodedTimeUnit::NANOS; - case ParquetTimeUnit::UNKNOWN: - default: - return DecodedTimeUnit::UNKNOWN; - } -} - -Status decoded_fixed_value_size(const std::string& column_name, DecodedValueKind value_kind, - size_t* value_size) { - switch (value_kind) { - case DecodedValueKind::BOOL: - *value_size = sizeof(bool); - return Status::OK(); - case DecodedValueKind::INT32: - *value_size = sizeof(int32_t); - return Status::OK(); - case DecodedValueKind::UINT32: - *value_size = sizeof(uint32_t); - return Status::OK(); - case DecodedValueKind::INT64: - *value_size = sizeof(int64_t); - return Status::OK(); - case DecodedValueKind::UINT64: - *value_size = sizeof(uint64_t); - return Status::OK(); - case DecodedValueKind::INT96: - *value_size = 12; - return Status::OK(); - case DecodedValueKind::FLOAT: - *value_size = sizeof(float); - return Status::OK(); - case DecodedValueKind::DOUBLE: - *value_size = sizeof(double); - return Status::OK(); - case DecodedValueKind::BINARY: - case DecodedValueKind::FIXED_BINARY: - return Status::InvalidArgument("Parquet binary value kind has no fixed value size for {}", - column_name); - } - return Status::InternalError("Unknown decoded value kind for column {}", column_name); -} - -Status get_binary_chunks(const std::string& column_name, - ::parquet::internal::RecordReader& record_reader, - std::vector>* chunks) { - if (auto* dictionary_reader = - dynamic_cast<::parquet::internal::DictionaryRecordReader*>(&record_reader); - dictionary_reader != nullptr) { - auto chunked = dictionary_reader->GetResult(); - if (chunked == nullptr) { - return Status::Corruption( - "Parquet dictionary record reader returned null result for column {}", - column_name); - } - *chunks = chunked->chunks(); - return Status::OK(); - } - auto* binary_reader = dynamic_cast<::parquet::internal::BinaryRecordReader*>(&record_reader); - if (binary_reader == nullptr) { - return Status::InternalError("Parquet binary record reader is not available for column {}", - column_name); - } - *chunks = binary_reader->GetBuilderChunks(); - return Status::OK(); -} - -Status append_dictionary_binary_values(const std::string& column_name, - const ::arrow::DictionaryArray& dictionary_array, - std::vector* values) { - DORIS_CHECK(values != nullptr); - const auto& dictionary = dictionary_array.dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - column_name); - } - auto append_value = [&](int64_t dictionary_index) -> Status { - if (dictionary_index < 0 || dictionary_index >= dictionary->length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(dictionary.get())) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(dictionary.get())) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back( - reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); - }; - for (int64_t row_idx = 0; row_idx < dictionary_array.length(); ++row_idx) { - if (dictionary_array.IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - RETURN_IF_ERROR(append_value(dictionary_array.GetValueIndex(row_idx))); - } - return Status::OK(); -} - -Status build_binary_values(const std::string& column_name, - const std::vector>& chunks, - int64_t records_read, const NullMap* null_map, - bool read_dense_for_nullable, std::vector* binary_values, - std::vector* compact_values) { - DORIS_CHECK(binary_values != nullptr); - DORIS_CHECK(compact_values != nullptr); - binary_values->clear(); - compact_values->clear(); - auto* values = read_dense_for_nullable ? compact_values : binary_values; - values->reserve(records_read); - for (const auto& chunk : chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", column_name); - } - if (auto* binary_array = dynamic_cast<::arrow::BinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < binary_array->length(); ++row_idx) { - if (binary_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(row_idx, &length); - values->emplace_back(reinterpret_cast(value), length); - } - } else if (auto* fixed_array = dynamic_cast<::arrow::FixedSizeBinaryArray*>(chunk.get())) { - for (int64_t row_idx = 0; row_idx < fixed_array->length(); ++row_idx) { - if (fixed_array->IsNull(row_idx)) { - values->emplace_back(static_cast(nullptr), 0); - continue; - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(row_idx)), - fixed_array->byte_width()); - } - } else if (auto* dictionary_array = dynamic_cast<::arrow::DictionaryArray*>(chunk.get())) { - RETURN_IF_ERROR( - append_dictionary_binary_values(column_name, *dictionary_array, values)); - } else { - return Status::InternalError("Unexpected Arrow binary array type for column {}", - column_name); - } - } - if (read_dense_for_nullable) { - if (null_map == nullptr || null_map->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid dense nullable parquet null map for column {}: rows={}, null_map={}", - column_name, records_read, null_map == nullptr ? 0 : null_map->size()); - } - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (compact_values->size() != static_cast(non_null_count)) { - return Status::Corruption( - "Invalid dense nullable parquet binary values for column {}: values={}, " - "records={}, nulls={}", - column_name, compact_values->size(), records_read, - records_read - non_null_count); - } - binary_values->reserve(records_read); - size_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - binary_values->emplace_back(static_cast(nullptr), 0); - continue; - } - binary_values->emplace_back((*compact_values)[value_idx++]); - } - return Status::OK(); - } - if (binary_values->size() != static_cast(records_read)) { - return Status::Corruption( - "Invalid parquet binary record read result for column {}: rows={}, records={}", - column_name, binary_values->size(), records_read); - } - return Status::OK(); -} - -float half_to_float(uint16_t value) { - const uint32_t sign = (value & 0x8000U) << 16; - const uint32_t exponent = (value & 0x7C00U) >> 10; - const uint32_t mantissa = value & 0x03FFU; - - if (exponent == 0) { - if (mantissa == 0) { - return std::bit_cast(sign); - } - const float subnormal = std::ldexp(static_cast(mantissa), -24); - return sign == 0 ? subnormal : -subnormal; - } - if (exponent == 0x1FU) { - return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); - } - return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); -} - -Status build_float16_values(const std::string& column_name, - const ParquetTypeDescriptor& type_descriptor, - const std::vector& binary_values, int64_t row_count, - std::vector* float_values) { - if (type_descriptor.fixed_length != 2) { - return Status::Corruption("Invalid parquet Float16 length for column {}: {}", column_name, - type_descriptor.fixed_length); - } - if (binary_values.size() != static_cast(row_count)) { - return Status::Corruption( - "Invalid parquet Float16 value count for column {}: values={}, rows={}", - column_name, binary_values.size(), row_count); - } - float_values->resize(static_cast(row_count)); - for (int64_t row = 0; row < row_count; ++row) { - const auto& binary_value = binary_values[static_cast(row)]; - if (binary_value.data == nullptr && binary_value.size == 0) { - (*float_values)[static_cast(row)] = 0; - continue; - } - if (binary_value.data == nullptr || binary_value.size != 2) { - return Status::Corruption( - "Invalid parquet Float16 value for column {} at row {}: data={}, size={}", - column_name, row, binary_value.data == nullptr ? "null" : "non-null", - binary_value.size); - } - uint16_t raw_value = 0; - std::memcpy(&raw_value, binary_value.data, sizeof(raw_value)); - (*float_values)[static_cast(row)] = half_to_float(raw_value); - } - return Status::OK(); -} - -} // namespace - -Status ParquetLeafReader::collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - batch->_values_written = record_reader.values_written(); - - if (!batch->is_binary_value()) { - batch->_fixed_values = record_reader.values(); - return Status::OK(); - } - - RETURN_IF_ERROR(get_binary_chunks(_name, record_reader, &batch->_binary_chunks)); - batch->_values_written = 0; - for (const auto& chunk : batch->_binary_chunks) { - if (chunk == nullptr) { - return Status::Corruption( - "Parquet binary record reader returned null chunk for column {}", _name); - } - batch->_values_written += chunk->length(); - } - return Status::OK(); -} - -Status ParquetLeafReader::collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const { - DORIS_CHECK(batch != nullptr); - batch->_def_levels = nullptr; - batch->_rep_levels = nullptr; - batch->_fixed_values = nullptr; - batch->_binary_chunks.clear(); - batch->_value_kind = decoded_value_kind(_type_descriptor); - batch->_consumed_level_count = record_reader.levels_position(); - batch->_decoded_level_count = record_reader.levels_written(); - if (_descriptor->max_definition_level() > 0) { - batch->_def_levels = record_reader.def_levels(); - } - if (_descriptor->max_repetition_level() > 0) { - batch->_rep_levels = record_reader.rep_levels(); - } - batch->_read_dense_for_nullable = record_reader.read_dense_for_nullable(); - - // Arrow's RecordReader::Reset() does not reset ByteArray/FLBA builders. GetBuilderChunks() - // (or DictionaryRecordReader::GetResult()) is the documented reset operation and must be - // called before the next ReadRecords(). Otherwise a levels-only skip followed by a normal read - // observes values from both batches; for example, skipping ARRAY ["a", "b"] and then - // reading ["c"] would report one current level but three values. Release the chunks here and - // let the temporary vector destroy them immediately. We deliberately do not inspect or copy - // their payload into a Doris Column, so the levels-only contract still avoids Doris-side value - // materialization. - if (batch->is_binary_value()) { - _discarded_binary_chunks.clear(); - auto status = get_binary_chunks(_name, record_reader, &_discarded_binary_chunks); - // GetBuilderChunks()/GetResult() transfers the Arrow builder result into shared_ptrs. - // Retaining those shared_ptrs in persistent scratch would keep the whole payload alive - // until the next levels-only batch and overlap it with the next builder allocation. - _discarded_binary_chunks.clear(); - RETURN_IF_ERROR(status); - } - - // COUNT(col) and nested skip only need top-level shape. Fixed-width values remain owned by the - // RecordReader and are cleared by Reset(); binary values were released above solely to reset - // the Arrow builder. - batch->_values_written = 0; - return Status::OK(); -} - -// - FLOAT16: binary -> half_to_float -> float_values -Status ParquetLeafReader::append_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, MutableColumnPtr& column) const { - return append_values_with_type(batch, row_count, null_map, _type, _serde, column); -} - -Status ParquetLeafReader::append_values_with_type(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, - const DataTypePtr& materialization_type, - const DataTypeSerDeSPtr& serde, - MutableColumnPtr& column) const { - DORIS_CHECK(materialization_type != nullptr); - DORIS_CHECK(serde != nullptr); - _binary_values.clear(); - _compact_binary_values.clear(); - _spaced_values.clear(); - _float_values.clear(); - Defer clear_logical_scratch([this] { - // StringRef entries borrow Arrow buffers owned by ParquetLeafBatch. Drop every borrowed - // pointer before the owner releases those chunks; clear() retains all reusable capacity. - _binary_values.clear(); - _compact_binary_values.clear(); - _spaced_values.clear(); - _float_values.clear(); - }); - DecodedColumnView view; - view.value_kind = batch._value_kind; - view.time_unit = decoded_time_unit(_type_descriptor.time_unit); - view.row_count = row_count; - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.decimal_precision = _type_descriptor.decimal_precision; - view.decimal_scale = _type_descriptor.decimal_scale; - view.fixed_length = _type_descriptor.fixed_length; - view.timestamp_is_adjusted_to_utc = _type_descriptor.timestamp_is_adjusted_to_utc; - view.timezone = _timezone; - view.enable_strict_mode = _enable_strict_mode; - view.null_map = null_map == nullptr || null_map->empty() ? nullptr : null_map->data(); - const bool read_dense_for_nullable = batch._read_dense_for_nullable && view.null_map != nullptr; - - if (_type_descriptor.extra_type_info == ParquetExtraTypeInfo::FLOAT16) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &_binary_values, - &_compact_binary_values)); - RETURN_IF_ERROR(build_float16_values(_name, _type_descriptor, _binary_values, row_count, - &_float_values)); - view.value_kind = DecodedValueKind::FLOAT; - view.values = reinterpret_cast(_float_values.data()); - } else if (batch.is_binary_value()) { - RETURN_IF_ERROR(build_binary_values(_name, batch._binary_chunks, row_count, null_map, - read_dense_for_nullable, &_binary_values, - &_compact_binary_values)); - view.binary_values = &_binary_values; - } else if (read_dense_for_nullable) { - RETURN_IF_ERROR(build_spaced_fixed_values(batch, row_count, null_map, &_spaced_values)); - view.values = _spaced_values.data(); - } else { - view.values = batch._fixed_values; - } - - if (_decoded_value_appender != nullptr) { - return _decoded_value_appender(column, view); - } - - { - SCOPED_TIMER(_profile.materialization_time); - if (!materialization_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& tmp_null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = tmp_null_map.size(); - auto st = serde->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - tmp_null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(tmp_null_map.data() + old_null_map_size, 0, - tmp_null_map.size() - old_null_map_size); - } else { - RETURN_IF_ERROR(serde->read_column_from_decoded_values(*column, view)); - } - } else { - RETURN_IF_ERROR(serde->read_column_from_decoded_values(*column, view)); - } - } - return Status::OK(); -} - -bool ParquetLeafBatch::is_binary_value() const { - return _value_kind == DecodedValueKind::BINARY || _value_kind == DecodedValueKind::FIXED_BINARY; -} - -Status ParquetLeafReader::build_spaced_fixed_values(const ParquetLeafBatch& batch, - int64_t row_count, const NullMap* null_map, - std::vector* spaced_values) const { - DORIS_CHECK(null_map != nullptr); - DORIS_CHECK(spaced_values != nullptr); - size_t value_size = 0; - RETURN_IF_ERROR(decoded_fixed_value_size(_name, batch._value_kind, &value_size)); - spaced_values->resize(static_cast(row_count) * value_size); - const auto non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(null_map->data()), null_map->size())); - if (batch._values_written != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet values for column {}: values={}, records={}, " - "nulls={}", - _name, batch._values_written, row_count, row_count - non_null_count); - } - auto* dst = spaced_values->data(); - int64_t value_idx = 0; - for (int64_t record_idx = 0; record_idx < row_count; ++record_idx) { - if ((*null_map)[record_idx] != 0) { - continue; // NULL row: skip it and keep the target slot zeroed - } - std::memcpy(dst + static_cast(record_idx) * value_size, - batch._fixed_values + static_cast(value_idx) * value_size, value_size); - ++value_idx; - } - return Status::OK(); -} - -ParquetLeafReader::ParquetLeafReader( - const ::parquet::ColumnDescriptor* descriptor, ParquetTypeDescriptor type_descriptor, - DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile, const cctz::time_zone* timezone, - bool enable_strict_mode, - std::function decoded_value_appender) - : _descriptor(descriptor), - _type_descriptor(type_descriptor), - _type(std::move(type)), - _name(std::move(name)), - _record_reader(std::move(record_reader)), - _profile(profile), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _decoded_value_appender(std::move(decoded_value_appender)), - _serde(_type->get_serde()), - _nested_value_type(remove_nullable(_type)), - _nested_value_serde(_nested_value_type->get_serde()) {} - -Status ParquetLeafReader::read_batch(int64_t batch_rows, ParquetLeafBatch* batch, - int64_t* rows_read) const { - if (batch == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet leaf batch result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - // A caller normally releases chunks immediately after consuming the batch. Clear once more at - // the producer boundary so an error in an earlier consumer can never overlap the previous - // Arrow payload with Reserve()/ReadRecords() for this batch. - batch->release_binary_chunks(); - - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - *rows_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet records for column {}: {}", _name, - e.what()); - } - if (*rows_read < 0 || *rows_read > batch_rows) { - return Status::Corruption("Invalid parquet record read result for column {}: {}", _name, - *rows_read); - } - auto status = collect_batch(*_record_reader, batch); - if (!status.ok()) { - // collect_batch() may already have acquired Arrow builder chunks before detecting a - // malformed chunk. No consumer will run on an error, so release that ownership here. - batch->release_binary_chunks(); - } - return status; -} - -Status ParquetLeafReader::build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const { - if (_descriptor->max_definition_level() == 0) { - return Status::OK(); - } - auto* def_levels = batch.def_levels(); - if (def_levels == nullptr && records_read > 0) { - return Status::Corruption( - "Parquet record reader returned null definition levels for nullable column {}", - _name); - } - const int16_t max_definition_level = _descriptor->max_definition_level(); - null_map->resize(records_read); - auto* __restrict dst = null_map->data(); - const auto* __restrict src = def_levels; - for (int64_t record_idx = 0; record_idx < records_read; ++record_idx) { - dst[record_idx] = src[record_idx] != max_definition_level; - } - return Status::OK(); -} - -Status ParquetLeafReader::read_nested_batch(int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - int64_t records_read = 0; - RETURN_IF_ERROR(read_batch(batch_rows, &_nested_leaf_batch, &records_read)); - Defer release_binary_chunks([this] { _nested_leaf_batch.release_binary_chunks(); }); - auto status = build_nested_batch_from_leaf_batch(_nested_leaf_batch, records_read, - value_slot_definition_level, batch, - value_slot_repetition_level); - return status; -} - -Status ParquetLeafReader::read_nested_levels_batch(int64_t batch_rows, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - - int64_t records_read = 0; - try { - _record_reader->Reset(); - _record_reader->Reserve(batch_rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - records_read = _record_reader->ReadRecords(batch_rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read parquet levels for column {}: {}", _name, - e.what()); - } - if (records_read < 0 || records_read > batch_rows) { - return Status::Corruption("Invalid parquet level read result for column {}: {}", _name, - records_read); - } - RETURN_IF_ERROR(collect_levels_batch(*_record_reader, &_nested_leaf_batch)); - return build_nested_levels_batch_from_leaf_batch(_nested_leaf_batch, records_read, batch); -} - -Status ParquetLeafReader::build_nested_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar batch is null for column {}", _name); - } - batch->reset(); - batch->value_slot_definition_level = value_slot_definition_level; - batch->value_slot_repetition_level = value_slot_repetition_level; - - batch->records_read = records_read; - if (_type->is_nullable() && leaf_batch.read_dense_for_nullable()) { - return Status::NotSupported( - "Dense nullable parquet nested reader is not supported for column {}", _name); - } - batch->levels_written = leaf_batch.consumed_level_count(); - const int64_t values_written = leaf_batch.values_written(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - if (batch->levels_written == 0 && batch->records_read > 0 && - values_written == batch->records_read && _descriptor->max_definition_level() == 0 && - _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read || values_written < 0 || - values_written > batch->levels_written) { - return Status::Corruption( - "Invalid nested parquet read result for column {}: rows={}, levels={}, values={}", - _name, batch->records_read, batch->levels_written, values_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - - const int16_t leaf_definition_level = _descriptor->max_definition_level(); - // Arrow's RecordReader may emit value placeholders for null ancestors that are below the - // Doris materialization threshold. Those slots must still advance the payload value index; - // otherwise the next defined child level points at the placeholder instead of its real value. - auto count_value_slots = [&](int16_t slot_definition_level) { - int64_t slot_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] >= slot_definition_level && - batch->rep_levels[level_idx] <= value_slot_repetition_level) { - ++slot_count; - } - } - return slot_count; - }; - - const int64_t value_slot_count = count_value_slots(value_slot_definition_level); - int16_t payload_slot_definition_level = value_slot_definition_level; - int64_t payload_value_slot_count = value_slot_count; - while (payload_slot_definition_level > 0 && payload_value_slot_count < values_written) { - --payload_slot_definition_level; - payload_value_slot_count = count_value_slots(payload_slot_definition_level); - } - - int64_t leaf_value_count = 0; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < value_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - if (batch->def_levels[level_idx] == leaf_definition_level) { - ++leaf_value_count; - } - } - - enum class ValueLayout { LEVELS, VALUE_SLOTS, LEAF_VALUES, PAYLOAD_VALUE_SLOTS }; - ValueLayout value_layout = ValueLayout::LEAF_VALUES; - if (values_written == batch->levels_written) { - value_layout = ValueLayout::LEVELS; - } else if (values_written == value_slot_count) { - value_layout = ValueLayout::VALUE_SLOTS; - } else if (values_written == leaf_value_count) { - value_layout = ValueLayout::LEAF_VALUES; - } else if (values_written == payload_value_slot_count) { - value_layout = ValueLayout::PAYLOAD_VALUE_SLOTS; - } else { - return Status::Corruption( - "Nested parquet reader returned inconsistent value count for column {}: values={}, " - "levels={}, slots={}, leaf_values={}, payload_slots={}, " - "payload_slot_definition_level={}", - _name, values_written, batch->levels_written, value_slot_count, leaf_value_count, - payload_value_slot_count, payload_slot_definition_level); - } - - batch->value_indices.resize(static_cast(batch->levels_written), -1); - _nested_value_nulls.resize(static_cast(values_written)); - std::fill(_nested_value_nulls.begin(), _nested_value_nulls.end(), 1); - int64_t value_idx = 0; - const int16_t decoded_slot_definition_level = value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS - ? payload_slot_definition_level - : value_slot_definition_level; - for (int64_t level_idx = 0; level_idx < batch->levels_written; ++level_idx) { - if (batch->def_levels[level_idx] < decoded_slot_definition_level || - batch->rep_levels[level_idx] > value_slot_repetition_level) { - continue; - } - const bool has_leaf_value = batch->def_levels[level_idx] == leaf_definition_level; - int64_t decoded_value_idx = -1; - if (value_layout == ValueLayout::LEVELS) { - decoded_value_idx = level_idx; - } else if (value_layout == ValueLayout::VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else if (value_layout == ValueLayout::PAYLOAD_VALUE_SLOTS) { - decoded_value_idx = value_idx++; - } else { - if (!has_leaf_value) { - continue; - } - decoded_value_idx = value_idx++; - } - DORIS_CHECK(decoded_value_idx >= 0); - DORIS_CHECK(decoded_value_idx < values_written); - if (has_leaf_value) { - batch->value_indices[static_cast(level_idx)] = decoded_value_idx; - _nested_value_nulls[static_cast(decoded_value_idx)] = 0; - } - } - if (value_layout != ValueLayout::LEVELS && value_idx != values_written) { - return Status::Corruption( - "Nested parquet reader value cursor stopped early for column {}: values={}, " - "visited={}", - _name, values_written, value_idx); - } - - if (batch->values_column.get() == nullptr) { - batch->values_column = _nested_value_type->create_column(); - } - if (values_written > 0) { - RETURN_IF_ERROR(append_values_with_type(leaf_batch, values_written, &_nested_value_nulls, - _nested_value_type, _nested_value_serde, - batch->values_column)); - } - return Status::OK(); -} - -Status ParquetLeafReader::build_nested_levels_batch_from_leaf_batch( - const ParquetLeafBatch& leaf_batch, int64_t records_read, - ParquetNestedScalarBatch* batch) const { - if (batch == nullptr) { - return Status::InvalidArgument("Nested scalar levels batch is null for column {}", _name); - } - batch->reset(); - batch->records_read = records_read; - batch->levels_written = leaf_batch.consumed_level_count(); - if (batch->levels_written > leaf_batch.decoded_level_count()) { - return Status::Corruption( - "Invalid nested parquet level position for column {}: position={}, levels={}", - _name, batch->levels_written, leaf_batch.decoded_level_count()); - } - - // Required flat leaves do not have physical def/rep level buffers. Synthesize one level slot - // per top-level row so the COUNT(col) aggregation code can use the same shape loop. - if (batch->levels_written == 0 && batch->records_read > 0 && - _descriptor->max_definition_level() == 0 && _descriptor->max_repetition_level() == 0) { - batch->levels_written = batch->records_read; - } - if (batch->levels_written < batch->records_read) { - return Status::Corruption( - "Invalid nested parquet levels result for column {}: rows={}, levels={}", _name, - batch->records_read, batch->levels_written); - } - if (batch->levels_written == 0) { - return Status::OK(); - } - - auto* def_levels = leaf_batch.def_levels(); - if (def_levels == nullptr && _descriptor->max_definition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null definition levels for column {}", _name); - } - batch->def_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_definition_level() == 0 || def_levels == nullptr) { - std::fill(batch->def_levels.begin(), batch->def_levels.end(), - _descriptor->max_definition_level()); - } else { - std::copy(def_levels, def_levels + batch->levels_written, batch->def_levels.begin()); - } - - auto* rep_levels = leaf_batch.rep_levels(); - if (rep_levels == nullptr && _descriptor->max_repetition_level() > 0) { - return Status::Corruption( - "Nested parquet reader returned null repetition levels for column {}", _name); - } - batch->rep_levels.resize(static_cast(batch->levels_written)); - if (_descriptor->max_repetition_level() == 0 || rep_levels == nullptr) { - std::fill(batch->rep_levels.begin(), batch->rep_levels.end(), 0); - } else { - std::copy(rep_levels, rep_levels + batch->levels_written, batch->rep_levels.begin()); - } - return Status::OK(); -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h b/be/src/format_v2/parquet/reader/parquet_leaf_reader.h deleted file mode 100644 index 6a5cb34254b465..00000000000000 --- a/be/src/format_v2/parquet/reader/parquet_leaf_reader.h +++ /dev/null @@ -1,213 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "common/status.h" -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type_serde/data_type_serde.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "core/string_ref.h" -#include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/parquet_type.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace arrow { -class Array; -} // namespace arrow - -namespace doris::format::parquet { - -struct ParquetLeafReaderTestAccess; - -// Read result for a nested scalar leaf, separating Dremel-encoded shape from actual values. -// The COUNT(col) aggregation fast path consumes only records_read, levels_written, def_levels, and rep_levels. -// That path does not populate value_indices or values_column, so callers must not call build_nested_column() afterwards. -struct ParquetNestedScalarBatch { - int64_t records_read = 0; - int64_t levels_written = 0; - int16_t value_slot_definition_level = 0; - int16_t value_slot_repetition_level = std::numeric_limits::max(); - std::vector def_levels; - std::vector rep_levels; - std::vector value_indices; - MutableColumnPtr values_column; - - bool empty() const { return levels_written == 0; } - - // Reset logical contents without replacing the vectors/Column object. A reader repeatedly - // sees the same physical leaf type, so retaining capacity is safe and avoids one allocation - // family per nested batch. A levels-only read leaves values_column empty but may keep the - // reusable object allocated. - void reset() { - records_read = 0; - levels_written = 0; - value_slot_definition_level = 0; - value_slot_repetition_level = std::numeric_limits::max(); - def_levels.clear(); - rep_levels.clear(); - value_indices.clear(); - if (values_column.get() != nullptr) { - values_column->clear(); - } - } -}; - -class ParquetLeafBatch { -public: - int64_t consumed_level_count() const { return _consumed_level_count; } - int64_t decoded_level_count() const { return _decoded_level_count; } - int64_t values_written() const { return _values_written; } - bool read_dense_for_nullable() const { return _read_dense_for_nullable; } - const int16_t* def_levels() const { return _def_levels; } - const int16_t* rep_levels() const { return _rep_levels; } - const std::vector>& binary_chunks() const { - return _binary_chunks; - } - - // Release Arrow payload ownership as soon as the synchronous Doris materialization finishes. - // clear() keeps vector capacity, so the batch remains allocation-reusable without extending - // the lifetime of what can be a very large BinaryArray/DictionaryArray payload. - void release_binary_chunks() { _binary_chunks.clear(); } - -private: - friend class ParquetLeafReader; - - bool is_binary_value() const; - - DecodedValueKind _value_kind = DecodedValueKind::INT32; - int64_t _consumed_level_count = 0; - int64_t _decoded_level_count = 0; - int64_t _values_written = 0; - const int16_t* _def_levels = nullptr; - const int16_t* _rep_levels = nullptr; - const uint8_t* _fixed_values = nullptr; - bool _read_dense_for_nullable = false; - std::vector> _binary_chunks; -}; - -// read_batch() -> build_null_map() + append_values() -// read_nested_batch() -class ParquetLeafReader { -public: - ParquetLeafReader(const ::parquet::ColumnDescriptor* descriptor, - ParquetTypeDescriptor type_descriptor, DataTypePtr type, std::string name, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile = {}, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - std::function - decoded_value_appender = nullptr); - - Status read_batch(int64_t batch_rows, ParquetLeafBatch* batch, int64_t* rows_read) const; - - Status build_null_map(const ParquetLeafBatch& batch, int64_t records_read, - NullMap* null_map) const; - - Status append_values(const ParquetLeafBatch& batch, int64_t row_count, const NullMap* null_map, - MutableColumnPtr& column) const; - - // LEVELS / VALUE_SLOTS / LEAF_VALUES / PAYLOAD_VALUE_SLOTS. - Status read_nested_batch( - int64_t batch_rows, int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level = std::numeric_limits::max()) const; - - // COUNT(col) and nested-skip shape-only read path. It still calls Arrow - // RecordReader::ReadRecords() to advance the Parquet cursor and obtain def/rep levels, but - // Doris only copies levels: - // - it does not build value_indices or values_column - // - it does not enter DataTypeSerde::read_column_from_decoded_values() - // - for Binary/FLBA, it releases and immediately discards Arrow builder chunks because that is - // the RecordReader's required reset operation; it never copies them into a Doris Column - // This lets COUNT(col) on MAP/ARRAY/STRUCT evaluate top-level NULL state and lets skip advance - // nested shape without Doris-side STRING/BINARY materialization. Arrow RecordReader does not - // expose a public levels-only API, so ReadRecords may still perform required page decoding. - Status read_nested_levels_batch(int64_t batch_rows, ParquetNestedScalarBatch* batch) const; - -private: - friend struct ParquetLeafReaderTestAccess; - - Status collect_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - // Levels-only variant of collect_batch(). It snapshots only def/rep level state and does not - // expose value buffers. Binary chunks are released only to reset Arrow's builder and are - // immediately discarded. Used by COUNT(col) and nested skip. - Status collect_levels_batch(::parquet::internal::RecordReader& record_reader, - ParquetLeafBatch* batch) const; - - Status build_spaced_fixed_values(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, - std::vector* spaced_values) const; - - Status append_values_with_type(const ParquetLeafBatch& batch, int64_t row_count, - const NullMap* null_map, const DataTypePtr& materialization_type, - const DataTypeSerDeSPtr& serde, MutableColumnPtr& column) const; - - Status build_nested_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - int16_t value_slot_definition_level, - ParquetNestedScalarBatch* batch, - int16_t value_slot_repetition_level) const; - Status build_nested_levels_batch_from_leaf_batch(const ParquetLeafBatch& leaf_batch, - int64_t records_read, - ParquetNestedScalarBatch* batch) const; - - const ::parquet::ColumnDescriptor* _descriptor = - nullptr; // Arrow column descriptor (physical_type, max_dl, max_rl) - ParquetTypeDescriptor - _type_descriptor; // type encoding information (decimal precision, timestamp unit, etc.) - DataTypePtr _type; // Doris target type - std::string _name; // column name for error messages - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader (shared ownership) - ParquetColumnReaderProfile _profile; // profile counters - const cctz::time_zone* _timezone = nullptr; // timezone for timestamp conversion - bool _enable_strict_mode = false; // strict mode for type mismatch errors - std::function _decoded_value_appender; - // Logical scratch. append_values() resets sizes but preserves capacity across batches. - // StringRef entries never outlive the Arrow chunks retained by the current leaf batch. - DataTypeSerDeSPtr _serde; - DataTypePtr _nested_value_type; - DataTypeSerDeSPtr _nested_value_serde; - mutable ParquetLeafBatch _nested_leaf_batch; - mutable NullMap _nested_value_nulls; - mutable std::vector _binary_values; - mutable std::vector _compact_binary_values; - mutable std::vector _spaced_values; - mutable std::vector _float_values; - mutable std::vector> _discarded_binary_chunks; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp index 4e9a363b13c7cb..dcd601b003ae3c 100644 --- a/be/src/format_v2/parquet/reader/row_position_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/row_position_column_reader.cpp @@ -20,6 +20,7 @@ #include "core/assert_cast.h" #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" +#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" namespace doris::format::parquet { diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp b/be/src/format_v2/parquet/reader/scalar_column_reader.cpp deleted file mode 100644 index 1d8a76f5d0de69..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.cpp +++ /dev/null @@ -1,596 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/scalar_column_reader.h" - -#include -#include -#include - -#include -#include -#include - -#include "core/column/column.h" -#include "core/column/column_nullable.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type_serde/decoded_column_view.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "util/defer_op.h" -#include "util/simd/bits.h" - -namespace doris::format::parquet { -namespace { - -class ParquetNestedScalarValueCursor { -public: - explicit ParquetNestedScalarValueCursor(const ParquetNestedScalarBatch* batch) { reset(batch); } - - void reset(const ParquetNestedScalarBatch* batch) { - DORIS_CHECK(batch != nullptr); - _batch = batch; - } - - Status value_index(const std::string& column_name, int64_t level_idx, int64_t* value_idx) { - DORIS_CHECK(_batch != nullptr); - DORIS_CHECK(value_idx != nullptr); - DORIS_CHECK(level_idx < _batch->levels_written); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(static_cast(level_idx) < _batch->value_indices.size()); - const int64_t computed_value_idx = _batch->value_indices[static_cast(level_idx)]; - if (computed_value_idx < 0) { - return Status::Corruption("Nested parquet value is absent for column {}", column_name); - } - DORIS_CHECK(_batch->values_column.get() != nullptr); - if (computed_value_idx >= _batch->values_column->size()) { - return Status::Corruption("Nested parquet value index is out of range for column {}", - column_name); - } - *value_idx = computed_value_idx; - return Status::OK(); - } - -private: - const ParquetNestedScalarBatch* _batch = nullptr; -}; - -Status append_scalar_batch_value(const ScalarColumnReader& column_reader, - const ParquetNestedScalarBatch& batch, int64_t level_idx, - ParquetNestedScalarValueCursor* value_cursor, - MutableColumnPtr& column) { - DORIS_CHECK(value_cursor != nullptr); - int64_t value_idx = -1; - RETURN_IF_ERROR(value_cursor->value_index(column_reader.name(), level_idx, &value_idx)); - auto* nullable_column = check_and_get_column(*column); - if (nullable_column != nullptr) { - nullable_column->get_nested_column().insert_from(*batch.values_column, - static_cast(value_idx)); - nullable_column->get_null_map_data().push_back(0); - return Status::OK(); - } - column->insert_from(*batch.values_column, static_cast(value_idx)); - return Status::OK(); -} - -Status append_arrow_binary_dictionary_value(const std::string& column_name, - const ::arrow::Array& dictionary, - int64_t dictionary_index, - std::vector* values) { - DORIS_CHECK(values != nullptr); - if (dictionary_index < 0 || dictionary_index >= dictionary.length()) { - return Status::Corruption("Invalid parquet dictionary index {} for column {}", - dictionary_index, column_name); - } - if (auto* binary_array = dynamic_cast(&dictionary)) { - if (binary_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - int32_t length = 0; - const uint8_t* value = binary_array->GetValue(dictionary_index, &length); - values->emplace_back(reinterpret_cast(value), length); - return Status::OK(); - } - if (auto* fixed_array = dynamic_cast(&dictionary)) { - if (fixed_array->IsNull(dictionary_index)) { - values->emplace_back(static_cast(nullptr), 0); - return Status::OK(); - } - values->emplace_back(reinterpret_cast(fixed_array->GetValue(dictionary_index)), - fixed_array->byte_width()); - return Status::OK(); - } - return Status::InternalError("Unexpected Arrow dictionary value array type for column {}", - column_name); -} - -} // namespace - -ScalarColumnReader::ScalarColumnReader( - const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan, const cctz::time_zone* timezone, - bool enable_strict_mode, ParquetColumnReaderProfile profile) - : ParquetColumnReader(column_schema, column_schema.type, profile), - _descriptor(column_schema.descriptor), - _type_descriptor(column_schema.type_descriptor), - _record_reader(std::move(record_reader)), - _leaf_reader(std::make_unique(_descriptor, _type_descriptor, _type, - _name, _record_reader, _profile, - timezone, enable_strict_mode)), - _page_skip_plan(page_skip_plan), - _timezone(timezone), - _enable_strict_mode(enable_strict_mode), - _nested_batch(std::make_unique()) {} - -ScalarColumnReader::~ScalarColumnReader() = default; - -Status ScalarColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("Invalid parquet column read result pointer for column {}", - _name); - } - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - auto& reader = leaf_reader(); - RETURN_IF_ERROR(reader.read_batch(rows, &_leaf_batch, rows_read)); - - // Every path below consumes the Arrow payload synchronously. Keep reusable vector capacity but - // release the shared_ptr ownership before this call returns, including validation failures. - Defer release_binary_chunks([this] { _leaf_batch.release_binary_chunks(); }); - - _null_map.clear(); - RETURN_IF_ERROR(reader.build_null_map(_leaf_batch, *rows_read, &_null_map)); - const auto value_kind = decoded_value_kind(_type_descriptor); - const bool is_binary_value = - value_kind == DecodedValueKind::BINARY || value_kind == DecodedValueKind::FIXED_BINARY; - if (!is_binary_value && _leaf_batch.read_dense_for_nullable() && !_null_map.empty()) { - const int64_t non_null_count = static_cast(simd::count_zero_num( - reinterpret_cast(_null_map.data()), _null_map.size())); - const int64_t null_count = *rows_read - non_null_count; - if (_leaf_batch.values_written() != non_null_count) { - return Status::Corruption( - "Invalid dense nullable parquet record read result for column {}: values={}, " - "records={}, nulls={}", - _name, _leaf_batch.values_written(), *rows_read, null_count); - } - } else if (!is_binary_value && !_leaf_batch.read_dense_for_nullable() && - _leaf_batch.values_written() != *rows_read) { - return Status::Corruption( - "Invalid parquet record read result for column {}: values={}, records={}", _name, - _leaf_batch.values_written(), *rows_read); - } - - RETURN_IF_ERROR(reader.append_values(_leaf_batch, *rows_read, &_null_map, column)); - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::skip_records(int64_t rows) { - if (_record_reader == nullptr) { - return Status::InternalError("Parquet record reader is not initialized for column {}", - _name); - } - if (rows <= 0) { - return Status::OK(); - } - int64_t skipped_rows = 0; - SCOPED_TIMER(_profile.arrow_skip_records_time); - try { - _record_reader->Reset(); - while (skipped_rows < rows) { - const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); - if (skipped <= 0) { - return Status::Corruption( - "Failed to skip parquet records for column {}: skipped {} of {} rows", - _name, skipped_rows, rows); - } - skipped_rows += skipped; - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to skip parquet records for column {}: {}", _name, - e.what()); - } - update_reader_skip_rows(rows); - return Status::OK(); -} - -int64_t ScalarColumnReader::page_filtered_rows_to_skip(int64_t rows) const { - if (_page_skip_plan == nullptr || rows <= 0) { - return 0; - } - const int64_t skip_end = _row_group_rows_read + rows; - int64_t filtered_rows = 0; - for (const auto& range : _page_skip_plan->skipped_ranges) { - const int64_t range_end = range.start + range.length; - if (range_end <= _row_group_rows_read) { - continue; - } - if (range.start >= skip_end) { - break; - } - const int64_t start = std::max(range.start, _row_group_rows_read); - const int64_t end = std::min(range_end, skip_end); - if (start < end) { - // Scheduler gap skips are derived from page-index selected_ranges. A page-filtered - // range can only overlap such a gap when the whole data page is outside every selected - // range, so partial overlap would mean the planner and scheduler are out of sync. - DORIS_CHECK(start == range.start); - DORIS_CHECK(end == range_end); - filtered_rows += end - start; - } - } - return filtered_rows; -} - -void ScalarColumnReader::advance_rows_read(int64_t rows) { - DORIS_CHECK(rows >= 0); - _row_group_rows_read += rows; -} - -Status ScalarColumnReader::skip(int64_t rows) { - if (rows <= 0) { - return Status::OK(); - } - - const int64_t page_filtered_rows = page_filtered_rows_to_skip(rows); - DORIS_CHECK(page_filtered_rows <= rows); - const int64_t record_reader_skip_rows = rows - page_filtered_rows; - RETURN_IF_ERROR(skip_records(record_reader_skip_rows)); - advance_rows_read(rows); - return Status::OK(); -} - -Status ScalarColumnReader::select_with_dictionary_filter(const SelectionVector& sel, - uint16_t selected_rows, int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, - bool* used_filter) { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(used_filter != nullptr); - RETURN_IF_ERROR(sel.verify(selected_rows, batch_rows)); - *used_filter = false; - row_filter->clear(); - row_filter->reserve(selected_rows); - DORIS_CHECK(_record_reader != nullptr); - // A clean fallback is possible only before any range skip or read advances the record reader. - // Once dictionary selection starts, losing dictionary output is corruption rather than a - // reason to retry through the ordinary selected-read path with an already advanced stream. - if (!_record_reader->read_dictionary()) { - return Status::OK(); - } - *used_filter = true; - - selection_to_ranges(sel, selected_rows, &_selection_ranges); - int64_t cursor = 0; - for (const auto& range : _selection_ranges) { - if (range.start < cursor || range.start + range.length > batch_rows) { - return Status::InvalidArgument( - "Invalid parquet dictionary selection range [{}, {}) for column {}", - range.start, range.start + range.length, _name); - } - RETURN_IF_ERROR(skip(range.start - cursor)); - - int64_t range_rows_read = 0; - RETURN_IF_ERROR(read_range_with_dictionary_filter(range.length, dictionary_filter, column, - row_filter, &range_rows_read, - used_filter)); - if (!*used_filter) { - return Status::OK(); - } - if (range_rows_read != range.length) { - return Status::Corruption( - "Parquet dictionary selected read returned {} rows, expected {} rows for " - "column {}", - range_rows_read, range.length, _name); - } - cursor = range.start + range.length; - } - RETURN_IF_ERROR(skip(batch_rows - cursor)); - if (_profile.reader_select_rows != nullptr) { - COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); - } - return Status::OK(); -} - -Status ScalarColumnReader::read_range_with_dictionary_filter( - int64_t rows, const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter) { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(rows_read != nullptr); - DORIS_CHECK(used_filter != nullptr); - DORIS_CHECK(_record_reader != nullptr); - if (!_record_reader->read_dictionary()) { - return Status::Corruption( - "Parquet dictionary reader became unavailable after selected reading started for " - "column {}", - _name); - } - - RETURN_IF_ERROR(leaf_reader().read_batch(rows, &_leaf_batch, rows_read)); - Defer release_binary_chunks([this] { _leaf_batch.release_binary_chunks(); }); - int64_t matched_rows = 0; - RETURN_IF_ERROR(append_dictionary_filtered_values(_leaf_batch.binary_chunks(), - dictionary_filter, column, row_filter, - &matched_rows, used_filter)); - if (!*used_filter) { - return Status::Corruption( - "Parquet dictionary reader did not return dictionary batches for column {}", _name); - } - if (row_filter->size() < static_cast(*rows_read)) { - return Status::Corruption( - "Parquet dictionary filter produced too few row decisions for column {}: " - "filter={}, rows={}", - _name, row_filter->size(), *rows_read); - } - advance_rows_read(*rows_read); - update_reader_read_rows(*rows_read); - return Status::OK(); -} - -Status ScalarColumnReader::append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { - DORIS_CHECK(row_filter != nullptr); - DORIS_CHECK(matched_rows != nullptr); - DORIS_CHECK(used_filter != nullptr); - *matched_rows = 0; - *used_filter = false; - - _dictionary_binary_values.clear(); - Defer clear_dictionary_binary_values([this] { - // StringRef entries borrow Arrow dictionary bytes. Clear them on every return path before - // read_range_with_dictionary_filter() releases the owning chunks. - _dictionary_binary_values.clear(); - }); - for (const auto& chunk : chunks) { - DORIS_CHECK(chunk != nullptr); - const auto* dict_array = dynamic_cast(chunk.get()); - if (dict_array == nullptr) { - // The caller has already consumed rows from a DictionaryRecordReader. Falling back to a - // normal selected read would desynchronize the Parquet stream, so absence of a - // DictionaryArray is reported as corruption by read_range_with_dictionary_filter(). - return Status::OK(); - } - *used_filter = true; - const auto& dictionary = dict_array->dictionary(); - if (dictionary == nullptr) { - return Status::Corruption("Parquet dictionary array has null dictionary for column {}", - _name); - } - - // Dictionary predicates are evaluated once against the dictionary page and produce a - // dictionary-entry bitmap. DATA_PAGE rows then only need an integer-index lookup. NULL rows - // do not have a dictionary entry and cannot satisfy the supported equality/IN predicates. - for (int64_t row = 0; row < dict_array->length(); ++row) { - bool keep = false; - if (!dict_array->IsNull(row)) { - const int64_t dictionary_index = dict_array->GetValueIndex(row); - if (dictionary_index < 0 || dictionary_index >= dictionary->length() || - dictionary_index >= static_cast(dictionary_filter.size())) { - return Status::Corruption( - "Invalid parquet dictionary index {} for column {}: dictionary={}, " - "filter={}", - dictionary_index, _name, dictionary->length(), - dictionary_filter.size()); - } - keep = dictionary_filter[static_cast(dictionary_index)] != 0; - if (keep) { - RETURN_IF_ERROR(append_arrow_binary_dictionary_value( - _name, *dictionary, dictionary_index, &_dictionary_binary_values)); - ++*matched_rows; - } - } - row_filter->push_back(keep ? 1 : 0); - } - } - - if (!*used_filter) { - return Status::OK(); - } - return append_decoded_binary_values(_dictionary_binary_values, column); -} - -Status ScalarColumnReader::append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const { - DecodedColumnView view; - view.value_kind = decoded_value_kind(_type_descriptor); - view.row_count = static_cast(values.size()); - view.logical_integer_bit_width = _type_descriptor.integer_bit_width; - view.logical_integer_is_signed = !_type_descriptor.is_unsigned_integer; - view.fixed_length = _type_descriptor.fixed_length; - view.binary_values = &values; - - SCOPED_TIMER(_profile.materialization_time); - if (!_type->is_nullable()) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - auto& nested_column = nullable_column->get_nested_column(); - auto& null_map = nullable_column->get_null_map_data(); - const auto old_nested_size = nested_column.size(); - const auto old_null_map_size = null_map.size(); - auto st = _type->get_serde()->read_column_from_decoded_values(nested_column, view); - if (!st.ok()) { - nested_column.resize(old_nested_size); - return st; - } - null_map.resize(old_null_map_size + nested_column.size() - old_nested_size); - memset(null_map.data() + old_null_map_size, 0, null_map.size() - old_null_map_size); - return Status::OK(); - } - return _type->get_serde()->read_column_from_decoded_values(*column, view); - } - - NullMap null_map(values.size(), 0); - view.null_map = null_map.empty() ? nullptr : null_map.data(); - return _type->get_serde()->read_column_from_decoded_values(*column, view); -} - -// The value index stream must advance on those null slots, otherwise later payload values shift. -Status ScalarColumnReader::load_nested_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - RETURN_IF_ERROR(leaf_reader().read_nested_batch(rows, materialized_slot_definition_level, - _nested_batch.get(), _repetition_level)); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::load_nested_levels_batch(int64_t rows) { - DORIS_CHECK(_nested_batch != nullptr); - reset_nested_build_level_cursor(); - RETURN_IF_ERROR(leaf_reader().read_nested_levels_batch(rows, _nested_batch.get())); - advance_rows_read(_nested_batch->records_read); - update_reader_read_rows(_nested_batch->records_read); - return Status::OK(); -} - -Status ScalarColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar build result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - // The levels-only loader intentionally does not populate value-slot metadata or payload - // buffers. Derive the logical slot threshold from the schema, exactly as load_nested_batch() - // does, so this consumer works for both loaded batch forms. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_read = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_read < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - if (def_level == _definition_level) { - RETURN_IF_ERROR(append_scalar_batch_value(*this, *_nested_batch, current_level_idx, - &value_cursor, column)); - } else { - if (!_type->is_nullable() && def_level >= _nullable_definition_level) { - return Status::Corruption( - "Parquet scalar column {} contains null for non-nullable field", _name); - } - column->insert_default(); - } - ++*values_read; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - if (values_consumed == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar consume result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - // A levels-only batch intentionally has no value-slot metadata. Reconstruct the same logical - // slot threshold used by load_nested_batch(): a nullable leaf owns a slot at one definition - // level below a non-null value, while a required leaf owns a slot only at its full definition - // level. For example, an empty ARRAY boundary must not be consumed as a STRING value. - const int16_t materialized_slot_definition_level = - static_cast(_definition_level - (_type->is_nullable() ? 1 : 0)); - *values_consumed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < _nested_batch->levels_written && *values_consumed < length_upper_bound) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = _nested_batch->def_levels[current_level_idx]; - const int16_t rep_level = _nested_batch->rep_levels[current_level_idx]; - ++level_idx; - if (def_level < materialized_slot_definition_level || rep_level > _repetition_level) { - continue; - } - RETURN_IF_ERROR(validate_nested_value(current_level_idx, false)); - ++*values_consumed; - } - set_nested_build_level_cursor(level_idx); - return Status::OK(); -} - -Status ScalarColumnReader::append_nested_value(int64_t level_idx, MutableColumnPtr& column) const { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet nested scalar append result for column {}", - _name); - } - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - ParquetNestedScalarValueCursor value_cursor(_nested_batch.get()); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return append_scalar_batch_value(*this, *_nested_batch, level_idx, &value_cursor, column); - } - if (!_type->is_nullable()) { - return Status::Corruption("Parquet MAP column {} contains null for non-nullable value", - _name); - } - column->insert_default(); - return Status::OK(); -} - -Status ScalarColumnReader::validate_nested_value(int64_t level_idx, bool require_non_null) const { - DORIS_CHECK(_nested_batch != nullptr); - DORIS_CHECK(level_idx >= 0); - DORIS_CHECK(level_idx < _nested_batch->levels_written); - const int16_t def_level = _nested_batch->def_levels[level_idx]; - if (def_level == _definition_level) { - return Status::OK(); - } - if (require_non_null || !_type->is_nullable()) { - return Status::Corruption("Parquet scalar column {} contains null for non-nullable field", - _name); - } - return Status::OK(); -} - -const std::vector& ScalarColumnReader::nested_definition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->def_levels; -} - -const std::vector& ScalarColumnReader::nested_repetition_levels() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->rep_levels; -} - -int64_t ScalarColumnReader::nested_levels_written() const { - DORIS_CHECK(_nested_batch != nullptr); - return _nested_batch->levels_written; -} - -bool ScalarColumnReader::is_or_has_repeated_child() const { - return _repetition_level > 0; -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/scalar_column_reader.h b/be/src/format_v2/parquet/reader/scalar_column_reader.h deleted file mode 100644 index ae611402aa0648..00000000000000 --- a/be/src/format_v2/parquet/reader/scalar_column_reader.h +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include - -#include "core/string_ref.h" -#include "format_v2/parquet/parquet_type.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -namespace parquet { -class ColumnDescriptor; - -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet - -namespace cctz { -class time_zone; -} // namespace cctz - -namespace doris::format::parquet { - -struct ScalarColumnReaderTestAccess; - -// Owns the physical leaf reader for the lifetime of the row-group reader. Keeping this object -// stable is important: ParquetLeafReader owns reusable SerDe and conversion scratch whose -// capacity would otherwise be discarded after every batch. -class ScalarColumnReader final : public ParquetColumnReader { - friend class MapColumnReader; - friend struct ScalarColumnReaderTestAccess; - -public: - ScalarColumnReader(const ParquetColumnSchema& column_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - const ParquetPageSkipPlan* page_skip_plan = nullptr, - const cctz::time_zone* timezone = nullptr, bool enable_strict_mode = false, - ParquetColumnReaderProfile profile = {}); - ~ScalarColumnReader() override; - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status select_with_dictionary_filter(const SelectionVector& sel, uint16_t selected_rows, - int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - bool* used_filter) override; - - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - -private: - Status append_nested_value(int64_t level_idx, MutableColumnPtr& column) const; - Status validate_nested_value(int64_t level_idx, bool require_non_null) const; - Status read_range_with_dictionary_filter(int64_t rows, const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, IColumn::Filter* row_filter, - int64_t* rows_read, bool* used_filter); - Status append_dictionary_filtered_values( - const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter); - Status append_decoded_binary_values(const std::vector& values, - MutableColumnPtr& column) const; - - const ::parquet::ColumnDescriptor* descriptor() const { return _descriptor; } - - ParquetLeafReader& leaf_reader() { - DORIS_CHECK(_leaf_reader != nullptr); - return *_leaf_reader; - } - - void advance_rows_read(int64_t rows); - Status skip_records(int64_t rows); - int64_t page_filtered_rows_to_skip(int64_t rows) const; - - const ::parquet::ColumnDescriptor* _descriptor = nullptr; // Arrow column descriptor - ParquetTypeDescriptor _type_descriptor; // type encoding information - std::shared_ptr<::parquet::internal::RecordReader> - _record_reader; // Arrow physical column reader - std::unique_ptr _leaf_reader; // persistent leaf decoder/materializer - const ParquetPageSkipPlan* _page_skip_plan = - nullptr; // page-index pruning result (may be nullptr) - const cctz::time_zone* _timezone = nullptr; // timezone - bool _enable_strict_mode = false; // strict mode - int64_t _row_group_rows_read = 0; // rows read in the current row group (cursor) - // Per-batch logical scratch. Each operation clears the logical size while retaining capacity. - ParquetLeafBatch _leaf_batch; - NullMap _null_map; - std::vector _selection_ranges; - std::vector _dictionary_binary_values; - std::unique_ptr _nested_batch; // intermediate result for nested reads -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.cpp b/be/src/format_v2/parquet/reader/struct_column_reader.cpp deleted file mode 100644 index 3df6437501934d..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.cpp +++ /dev/null @@ -1,287 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/struct_column_reader.h" - -#include -#include -#include -#include - -#include "core/column/column_struct.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" - -namespace doris::format::parquet { - -ParquetColumnReader* StructColumnReader::shape_source_reader() const { - for (const auto& child : _children) { - auto* child_reader = child.get(); - DORIS_CHECK(child_reader != nullptr); - if (!child_reader->is_or_has_repeated_child()) { - return child_reader; - } - } - if (_children.empty()) { - return nullptr; - } - return _children[0].get(); -} - -Status StructColumnReader::advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const { - DORIS_CHECK(child_reader != nullptr); - const int64_t next_child_cursor = parent_level_idx + 1; - if (auto* scalar_child = dynamic_cast(child_reader)) { - if (next_child_cursor > scalar_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - scalar_child->name(), _name); - } - scalar_child->set_nested_build_level_cursor( - std::max(scalar_child->nested_build_level_cursor(), next_child_cursor)); - return Status::OK(); - } - if (auto* struct_child = dynamic_cast(child_reader); - struct_child != nullptr && !struct_child->is_or_has_repeated_child()) { - if (next_child_cursor > struct_child->nested_levels_written()) { - return Status::Corruption( - "Parquet STRUCT child {} ended before null parent row in column {}", - struct_child->name(), _name); - } - struct_child->set_nested_build_level_cursor( - std::max(struct_child->nested_build_level_cursor(), next_child_cursor)); - for (auto& grandchild : struct_child->_children) { - RETURN_IF_ERROR(struct_child->advance_child_past_null_parent(grandchild.get(), - parent_level_idx)); - } - return Status::OK(); - } - - int64_t child_cursor = child_reader->nested_build_level_cursor(); - const auto& child_rep_levels = child_reader->nested_repetition_levels(); - const int64_t child_levels_written = child_reader->nested_levels_written(); - while (child_cursor < child_levels_written) { - const int16_t child_rep_level = child_rep_levels[child_cursor]; - ++child_cursor; - if (!child_reader->is_or_has_repeated_child() || child_rep_level <= _repetition_level) { - break; - } - } - child_reader->set_nested_build_level_cursor(child_cursor); - return Status::OK(); -} - -Status StructColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) { - RETURN_IF_ERROR(load_nested_batch(rows)); - return build_nested_column(rows, column, rows_read); -} - -Status StructColumnReader::skip(int64_t rows) { - return skip_nested_rows(rows); -} - -Status StructColumnReader::load_nested_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::load_nested_levels_batch(int64_t rows) { - reset_nested_build_level_cursor(); - for (auto& child_reader : _children) { - DORIS_CHECK(child_reader != nullptr); - RETURN_IF_ERROR(child_reader->load_nested_levels_batch(rows)); - } - return Status::OK(); -} - -Status StructColumnReader::build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) { - if (column.get() == nullptr) { - return Status::InvalidArgument("Invalid parquet struct build result pointer for column {}", - _name); - } - return _consume_or_build_nested_column(length_upper_bound, &column, values_read); -} - -Status StructColumnReader::consume_nested_column(int64_t length_upper_bound, - int64_t* values_consumed) { - return _consume_or_build_nested_column(length_upper_bound, nullptr, values_consumed); -} - -Status StructColumnReader::_consume_or_build_nested_column(int64_t length_upper_bound, - MutableColumnPtr* column, - int64_t* values_processed) { - if (values_processed == nullptr) { - return Status::InvalidArgument( - "Invalid parquet struct process result pointer for column {}", _name); - } - if (_children.empty()) { - if (column != nullptr) { - (*column)->resize((*column)->size() + static_cast(length_upper_bound)); - } - *values_processed = length_upper_bound; - return Status::OK(); - } - ColumnStruct* struct_column = nullptr; - NullMap* parent_null_map = nullptr; - if (column != nullptr) { - struct_column = struct_column_from_output(*column); - DORIS_CHECK(struct_column != nullptr); - parent_null_map = null_map_from_nullable_output(*column); - } - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - const auto& def_levels = shape_reader->nested_definition_levels(); - const auto& rep_levels = shape_reader->nested_repetition_levels(); - const int64_t levels_written = shape_reader->nested_levels_written(); - - _parent_nulls.clear(); - _parent_level_indices.clear(); - *values_processed = 0; - int64_t level_idx = nested_build_level_cursor(); - while (level_idx < levels_written) { - const int64_t current_level_idx = level_idx; - const int16_t def_level = def_levels[level_idx]; - const int16_t rep_level = rep_levels[level_idx]; - const bool starts_parent = - !shape_reader->is_or_has_repeated_child() || rep_level <= _repetition_level; - if (starts_parent && *values_processed >= length_upper_bound) { - break; - } - ++level_idx; - if (def_level < _repeated_ancestor_definition_level) { - continue; - } - if (shape_reader->is_or_has_repeated_child() && rep_level > _repetition_level) { - continue; - } - const bool parent_is_null = def_level < _nullable_definition_level; - if (parent_is_null && !_type->is_nullable()) { - return Status::Corruption( - "Parquet STRUCT column {} contains null for non-nullable struct", _name); - } - _parent_nulls.push_back(parent_is_null); - _parent_level_indices.push_back(current_level_idx); - ++*values_processed; - } - set_nested_build_level_cursor(level_idx); - - _child_columns.clear(); - if (column != nullptr) { - _child_columns.reserve(struct_column->get_columns().size()); - for (size_t child_idx = 0; child_idx < struct_column->get_columns().size(); ++child_idx) { - _child_columns.push_back(struct_column->get_column_ptr(child_idx)->assert_mutable()); - } - } - for (size_t child_idx = 0; child_idx < _children.size(); ++child_idx) { - const int output_idx = _child_output_indices[child_idx]; - if (column != nullptr && output_idx < 0) { - continue; - } - // STRUCT owns row alignment. Child readers consume only present parent rows from their - // level streams; null STRUCT parents become default placeholders in every child column. - // This mirrors Arrow's separation between struct validity and child array materialization, - // and avoids asking scalar/list/map children to invent values for an absent parent. - int64_t pending_present_rows = 0; - int64_t total_child_rows = 0; - auto flush_present_rows = [&]() -> Status { - if (pending_present_rows == 0) { - return Status::OK(); - } - int64_t child_rows = 0; - if (column != nullptr) { - RETURN_IF_ERROR(_children[child_idx]->build_nested_column( - pending_present_rows, _child_columns[output_idx], &child_rows)); - } else { - RETURN_IF_ERROR(_children[child_idx]->consume_nested_column(pending_present_rows, - &child_rows)); - } - if (child_rows != pending_present_rows) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), child_rows, pending_present_rows, _name); - } - total_child_rows += child_rows; - pending_present_rows = 0; - return Status::OK(); - }; - for (size_t parent_idx = 0; parent_idx < _parent_nulls.size(); ++parent_idx) { - const auto parent_is_null = _parent_nulls[parent_idx]; - if (!parent_is_null) { - ++pending_present_rows; - continue; - } - RETURN_IF_ERROR(flush_present_rows()); - if (column != nullptr) { - _child_columns[output_idx]->insert_default(); - } - RETURN_IF_ERROR(advance_child_past_null_parent(_children[child_idx].get(), - _parent_level_indices[parent_idx])); - ++total_child_rows; - } - RETURN_IF_ERROR(flush_present_rows()); - if (total_child_rows != *values_processed) { - return Status::Corruption( - "Parquet STRUCT child {} built {} rows, expected {} for column {}", - _children[child_idx]->name(), total_child_rows, *values_processed, _name); - } - } - if (column != nullptr) { - for (size_t child_idx = 0; child_idx < _child_columns.size(); ++child_idx) { - struct_column->get_column_ptr(child_idx) = std::move(_child_columns[child_idx]); - } - append_parent_nulls(parent_null_map, _parent_nulls); - } - return Status::OK(); -} - -const std::vector& StructColumnReader::nested_definition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_definition_levels(); -} - -const std::vector& StructColumnReader::nested_repetition_levels() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_repetition_levels(); -} - -int64_t StructColumnReader::nested_levels_written() const { - auto* shape_reader = shape_source_reader(); - DORIS_CHECK(shape_reader != nullptr); - return shape_reader->nested_levels_written(); -} - -bool StructColumnReader::is_or_has_repeated_child() const { - auto* shape_reader = shape_source_reader(); - return shape_reader != nullptr && shape_reader->is_or_has_repeated_child(); -} - -void StructColumnReader::advance_nested_build_level_cursor_past_parent( - int16_t parent_repetition_level) { - ParquetColumnReader::advance_nested_build_level_cursor_past_parent(parent_repetition_level); - for (auto& child : _children) { - DORIS_CHECK(child != nullptr); - child->advance_nested_build_level_cursor_past_parent(parent_repetition_level); - } -} - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/struct_column_reader.h b/be/src/format_v2/parquet/reader/struct_column_reader.h deleted file mode 100644 index 57e66e1e2236fd..00000000000000 --- a/be/src/format_v2/parquet/reader/struct_column_reader.h +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include - -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { - -class StructColumnReader final : public ParquetColumnReader { -public: - StructColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, - std::vector> children, - std::vector child_output_indices, - ParquetColumnReaderProfile profile = {}) - : ParquetColumnReader(schema, type, profile), - _children(std::move(children)), - _child_output_indices(std::move(child_output_indices)) { - DCHECK_EQ(_children.size(), _child_output_indices.size()); - } - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override; - Status skip(int64_t rows) override; - Status load_nested_batch(int64_t rows) override; - Status load_nested_levels_batch(int64_t rows) override; - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override; - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override; - const std::vector& nested_definition_levels() const override; - const std::vector& nested_repetition_levels() const override; - int64_t nested_levels_written() const override; - bool is_or_has_repeated_child() const override; - void advance_nested_build_level_cursor_past_parent(int16_t parent_repetition_level) override; - -private: - Status _consume_or_build_nested_column(int64_t length_upper_bound, MutableColumnPtr* column, - int64_t* values_processed); - ParquetColumnReader* shape_source_reader() const; - Status advance_child_past_null_parent(ParquetColumnReader* child_reader, - int64_t parent_level_idx) const; - - std::vector> _children; // projected child readers - std::vector _child_output_indices; // child reader -> struct output position mapping - // Parent shape is derived once from the chosen shape child, then reused for every projected - // child. These buffers preserve capacity across batches and avoid allocating per child. - NullMap _parent_nulls; - std::vector _parent_level_indices; - std::vector _child_columns; -}; - -} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index 8d3a3dd4accfa4..b77a1198d22ccb 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -33,11 +33,11 @@ struct RowRange { struct ParquetPageSkipPlan { int leaf_column_id = -1; // Page ordinal is the data-page ordinal in the column chunk. It intentionally excludes - // dictionary pages, matching Arrow PageReader::set_data_page_filter(). + // dictionary pages, matching the native page reader's ordinal domain. std::vector skipped_pages; std::vector skipped_page_compressed_sizes; - // Row ranges covered by skipped data pages. ScalarColumnReader uses these ranges to avoid - // calling RecordReader::SkipRecords() again for pages already skipped by Arrow. + // Row ranges covered by skipped data pages. NativeColumnReader uses these ranges to avoid + // consuming logical rows twice after the page reader has already skipped their payload. std::vector skipped_ranges; bool empty() const { return skipped_ranges.empty(); } diff --git a/be/test/format_v2/parquet/parquet_column_reader_test.cpp b/be/test/format_v2/parquet/parquet_column_reader_test.cpp deleted file mode 100644 index fb4cd129e1b03d..00000000000000 --- a/be/test/format_v2/parquet/parquet_column_reader_test.cpp +++ /dev/null @@ -1,3682 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_decimal.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_struct.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_struct.h" -#include "core/types.h" -#include "format_v2/file_reader.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" -#include "format_v2/parquet/selection_vector.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -template -const ColumnType& get_nullable_nested_column(const IColumn& column) { - // File-local schema exposed by the parquet reader follows Doris external-table semantics: - // nested STRUCT fields, LIST elements, and MAP keys/values are nullable even when the parquet - // field is required. - const auto& nullable_column = assert_cast(column); - return assert_cast(nullable_column.get_nested_column()); -} - -ParquetColumnSchema mock_column_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "mock"; - schema.type = std::make_shared(); - return schema; -} - -class BaseUnsupportedReader final : public ParquetColumnReader { -public: - BaseUnsupportedReader() - : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } -}; - -class DefaultSelectReader final : public ParquetColumnReader { -public: - DefaultSelectReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - auto& values = assert_cast(*column); - for (int64_t row = 0; row < rows; ++row) { - values.insert_value(static_cast(_cursor + row)); - } - _cursor += rows; - *rows_read = rows; - _read_ranges.push_back(rows); - return Status::OK(); - } - - Status skip(int64_t rows) override { - _cursor += rows; - _skip_ranges.push_back(rows); - return Status::OK(); - } - - const std::vector& read_ranges() const { return _read_ranges; } - const std::vector& skip_ranges() const { return _skip_ranges; } - -private: - int64_t _cursor = 0; - std::vector _read_ranges; - std::vector _skip_ranges; -}; - -class NestedSkipReader final : public ParquetColumnReader { -public: - NestedSkipReader() : ParquetColumnReader(mock_column_schema(), mock_column_schema().type) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { return Status::OK(); } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - *values_consumed = length_upper_bound; - return Status::OK(); - } -}; - -class ParquetColumnReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_column_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "reader.parquet").string(); - _plain_file_path = (_test_dir / "plain_reader.parquet").string(); - write_parquet_file(); - _file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto metadata = _file_reader->metadata(); - ASSERT_EQ(metadata->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(_fields.size(), _expected_by_field.size()); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_all_null_int32_array() { - arrow::Int32Builder builder; - for (int64_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(builder.AppendNull().ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_required_struct_array() { - auto struct_type = arrow::struct_({arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - const std::vector a_values = {101, 102, 103, 104, 105}; - const std::vector b_values = {"sa", "sb", "sc", "sd", "se"}; - for (size_t row = 0; row < a_values.size(); ++row) { - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(a_values[row]).ok()); - EXPECT_TRUE(b_builder->Append(b_values[row]).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* b_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(201).ok()); - EXPECT_TRUE(b_builder->Append("nsa").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(203).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(204).ok()); - EXPECT_TRUE(b_builder->Append("nsd").ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_decimal_array() { - auto decimal_type = arrow::decimal128(38, 6); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("d", decimal_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto d_array_builder = std::make_unique( - decimal_type, arrow::default_memory_pool()); - field_builders.push_back(std::shared_ptr(std::move(d_array_builder))); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - auto* d_builder = assert_cast(builder.field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(123456789)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(d_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(d_builder->Append(arrow::Decimal128(-987654321)).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("xs", list_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - field_builders.push_back(list_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(301).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(303).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(304).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(305).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_map_array() { - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("kv", map_type, true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - field_builders.push_back(map_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - auto* a_builder = assert_cast(builder.field_builder(0)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(401).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("one").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(403).ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(404).ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(a_builder->Append(405).ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->Append("five").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_with_nested_struct_list_array() { - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto nested_type = arrow::struct_({arrow::field("xs", list_type, true)}); - auto struct_type = arrow::struct_({arrow::field("nested", nested_type, true)}); - - auto value_builder = std::make_shared(); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - std::vector> nested_field_builders; - nested_field_builders.push_back(list_builder); - auto nested_builder = std::make_shared( - nested_type, arrow::default_memory_pool(), std::move(nested_field_builders)); - std::vector> field_builders; - field_builders.push_back(nested_builder); - arrow::StructBuilder builder(struct_type, arrow::default_memory_pool(), - std::move(field_builders)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(7).ok()); - EXPECT_TRUE(value_builder->Append(8).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(nested_builder->Append().ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), false))); - const std::vector> values = { - {1, 2}, {3}, {4, 5, 6}, {7}, {8, 9}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto value : row) { - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append(20).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(30).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_int_list_array() { - auto value_builder = std::make_shared(); - arrow::ListBuilder builder(arrow::default_memory_pool(), value_builder, - arrow::list(arrow::field("element", arrow::int32(), true))); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(110).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(120).ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(value_builder->Append(130).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_struct_list_array() { - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - auto* a_builder = assert_cast(struct_builder->field_builder(0)); - auto* b_builder = assert_cast(struct_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(11).ok()); - EXPECT_TRUE(b_builder->Append("la").ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(12).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(13).ok()); - EXPECT_TRUE(b_builder->Append("ld").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(14).ok()); - EXPECT_TRUE(b_builder->Append("le").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_list_list_int_array() { - auto value_builder = std::make_shared(); - auto inner_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto inner_list_builder = std::make_shared( - arrow::default_memory_pool(), value_builder, inner_list_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), inner_list_builder, - arrow::list(arrow::field("element", inner_list_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->AppendNull().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(inner_list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(inner_list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(5).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - const std::vector>> values = { - {{1, "a"}, {2, "b"}}, {{3, "c"}}, {{4, "d"}, {5, "e"}, {6, "f"}}, - {{7, "g"}}, {{8, "h"}, {9, "i"}}, - }; - for (const auto& row : values) { - EXPECT_TRUE(builder.Append().ok()); - for (const auto& [key, value] : row) { - EXPECT_TRUE(key_builder->Append(key).ok()); - EXPECT_TRUE(value_builder->Append(value).ok()); - } - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_required_nullable_string_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append("bb").ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->Append("cc").ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_struct_map_array() { - auto key_builder = std::make_shared(); - auto struct_type = arrow::struct_( - {arrow::field("a", arrow::int32(), false), arrow::field("b", arrow::utf8(), true)}); - std::vector> field_builders; - auto a_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(a_array_builder))); - auto b_array_builder = std::make_unique(); - field_builders.push_back(std::shared_ptr(std::move(b_array_builder))); - auto value_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(field_builders)); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", struct_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, value_builder, - map_type); - auto* a_builder = assert_cast(value_builder->field_builder(0)); - auto* b_builder = assert_cast(value_builder->field_builder(1)); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(101).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(21).ok()); - EXPECT_TRUE(b_builder->Append("ma").ok()); - EXPECT_TRUE(key_builder->Append(102).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(22).ok()); - EXPECT_TRUE(b_builder->AppendNull().ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(103).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(104).ok()); - EXPECT_TRUE(value_builder->Append().ok()); - EXPECT_TRUE(a_builder->Append(24).ok()); - EXPECT_TRUE(b_builder->Append("me").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_list_map_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - value_builder, list_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(201).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append(2).ok()); - EXPECT_TRUE(key_builder->Append(202).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(203).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(204).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(value_builder->Append(3).ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(205).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(value_builder->Append(4).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_map_list_array() { - auto key_builder = std::make_shared(); - auto value_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto map_builder = std::make_shared( - arrow::default_memory_pool(), key_builder, value_builder, map_type); - arrow::ListBuilder builder(arrow::default_memory_pool(), map_builder, - arrow::list(arrow::field("element", map_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(value_builder->Append("a").ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(value_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(value_builder->Append("d").ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int_map_map_array() { - auto key_builder = std::make_shared(); - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, nested_map_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(101).ok()); - EXPECT_TRUE(nested_value_builder->Append("aa").ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(401).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_list_struct_map_list_array() { - auto element_builder = std::make_shared(); - auto list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - element_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - auto map_builder = std::make_shared(arrow::default_memory_pool(), - key_builder, list_builder, map_type); - auto struct_type = arrow::struct_({arrow::field("kv", map_type, true)}); - std::vector> struct_field_builders; - struct_field_builders.push_back(map_builder); - auto struct_builder = std::make_shared( - struct_type, arrow::default_memory_pool(), std::move(struct_field_builders)); - arrow::ListBuilder builder(arrow::default_memory_pool(), struct_builder, - arrow::list(arrow::field("element", struct_type, true))); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(1).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(10).ok()); - EXPECT_TRUE(element_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(2).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(struct_builder->AppendNull().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendNull().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(struct_builder->Append().ok()); - EXPECT_TRUE(map_builder->Append().ok()); - EXPECT_TRUE(key_builder->Append(3).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(4).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(element_builder->Append(40).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_deep_map_list_map_array() { - auto nested_key_builder = std::make_shared(); - auto nested_value_builder = std::make_shared(); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto nested_map_builder = std::make_shared( - arrow::default_memory_pool(), nested_key_builder, nested_value_builder, - nested_map_type); - auto list_type = arrow::list(arrow::field("element", nested_map_type, true)); - auto list_builder = std::make_shared(arrow::default_memory_pool(), - nested_map_builder, list_type); - auto key_builder = std::make_shared(); - auto map_type = arrow::map(arrow::int32(), arrow::field("value", list_type, true)); - arrow::MapBuilder builder(arrow::default_memory_pool(), key_builder, list_builder, - map_type); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(10).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(1).ok()); - EXPECT_TRUE(nested_value_builder->Append("a").ok()); - EXPECT_TRUE(nested_key_builder->Append(2).ok()); - EXPECT_TRUE(nested_value_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->AppendEmptyValue().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(20).ok()); - EXPECT_TRUE(list_builder->AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.AppendEmptyValue().ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(30).ok()); - EXPECT_TRUE(list_builder->AppendNull().ok()); - EXPECT_TRUE(key_builder->Append(40).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(3).ok()); - EXPECT_TRUE(nested_value_builder->Append("c").ok()); - - EXPECT_TRUE(builder.Append().ok()); - EXPECT_TRUE(key_builder->Append(50).ok()); - EXPECT_TRUE(list_builder->Append().ok()); - EXPECT_TRUE(nested_map_builder->AppendNull().ok()); - EXPECT_TRUE(nested_map_builder->Append().ok()); - EXPECT_TRUE(nested_key_builder->Append(4).ok()); - EXPECT_TRUE(nested_value_builder->Append("d").ok()); - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, std::shared_ptr array, - std::function validator) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - _expected_by_field.push_back(std::move(validator)); - } - - void write_parquet_file() { - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"}), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); - add_field(arrow::field("all_null_int_col", arrow::int32(), true), - build_all_null_int32_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - for (size_t row = 0; row < ROW_COUNT; ++row) { - EXPECT_TRUE(nullable_column.is_null_at(row)); - } - }); - add_field(arrow::field("struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), false), - }), - false), - build_required_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_STRUCT); - const auto& struct_column = assert_cast(column); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - get_nullable_nested_column(struct_column.get_column(1)); - EXPECT_EQ(a_values.get_element(0), 101); - EXPECT_EQ(a_values.get_element(4), 105); - EXPECT_EQ(b_values.get_data_at(1).to_string(), "sb"); - EXPECT_EQ(b_values.get_data_at(4).to_string(), "se"); - }); - add_field(arrow::field("nullable_struct_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }), - true), - build_nullable_struct_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_nested = - assert_cast(b_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 201); - EXPECT_EQ(a_values.get_element(2), 203); - EXPECT_EQ(a_values.get_element(3), 204); - EXPECT_FALSE(b_values.is_null_at(0)); - EXPECT_TRUE(b_values.is_null_at(2)); - EXPECT_FALSE(b_values.is_null_at(3)); - EXPECT_EQ(b_nested.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(b_nested.get_data_at(3).to_string(), "nsd"); - }); - add_field(arrow::field("nullable_struct_decimal_col", - arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("d", arrow::decimal128(38, 6), true), - }), - true), - build_nullable_struct_with_decimal_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& d_values = - assert_cast(struct_column.get_column(1)); - const auto& d_nested = - assert_cast(d_values.get_nested_column()); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_FALSE(d_values.is_null_at(0)); - EXPECT_TRUE(d_values.is_null_at(2)); - EXPECT_FALSE(d_values.is_null_at(3)); - EXPECT_EQ(d_nested.get_element(0), Decimal128V3(123456789)); - EXPECT_EQ(d_nested.get_element(3), Decimal128V3(-987654321)); - }); - auto struct_list_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("xs", arrow::list(arrow::field("element", arrow::int32(), true)), - true), - }); - add_field(arrow::field("nullable_struct_list_col", struct_list_type, true), - build_nullable_struct_with_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(2), 303); - EXPECT_EQ(a_values.get_element(3), 304); - EXPECT_EQ(a_values.get_element(4), 305); - - const auto& xs_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = - assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = - assert_cast(xs_array.get_data()); - ASSERT_EQ(elements.size(), 4); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - const auto& values = - assert_cast(elements.get_nested_column()); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_EQ(values.get_element(3), 5); - }); - auto struct_map_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("kv", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - }); - add_field(arrow::field("nullable_struct_map_col", struct_map_type, true), - build_nullable_struct_with_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 2); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(2), 403); - EXPECT_EQ(a_values.get_element(3), 404); - EXPECT_EQ(a_values.get_element(4), 405); - - const auto& kv_nullable = - assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = - assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); - }); - auto nested_struct_list_type = arrow::struct_({ - arrow::field("nested", - arrow::struct_({ - arrow::field("xs", - arrow::list(arrow::field("element", - arrow::int32(), true)), - true), - }), - true), - }); - add_field(arrow::field("nullable_struct_nested_struct_list_col", nested_struct_list_type, - true), - build_nullable_struct_with_nested_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = - assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); - }); - add_field(arrow::field("list_int_col", - arrow::list(arrow::field("element", arrow::int32(), false)), false), - build_required_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_ARRAY); - const auto* array_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ( - remove_nullable(array_type->get_nested_type())->get_primitive_type(), - TYPE_INT); - const auto& array_column = assert_cast(column); - ASSERT_EQ(array_column.size(), ROW_COUNT); - const auto array_size_at = [&array_column](size_t row_idx) { - return array_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : array_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(array_size_at(0), 2); - EXPECT_EQ(array_size_at(1), 1); - EXPECT_EQ(array_size_at(2), 3); - EXPECT_EQ(array_size_at(4), 2); - const auto& values = - get_nullable_nested_column(array_column.get_data()); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(5), 6); - EXPECT_EQ(values.get_element(8), 9); - }); - add_field(arrow::field("nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), true), - build_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 30); - EXPECT_EQ(values.get_element(4), 40); - }); - add_field(arrow::field("required_nullable_list_int_col", - arrow::list(arrow::field("element", arrow::int32(), true)), false), - build_required_nullable_int_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& array_column = assert_cast(column); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 5); - EXPECT_EQ(offsets[4], 5); - const auto& elements = - assert_cast(array_column.get_data()); - ASSERT_EQ(elements.size(), 5); - EXPECT_TRUE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(4)); - }); - auto list_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field("nullable_list_struct_col", - arrow::list(arrow::field("element", list_struct_type, true)), true), - build_nullable_struct_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& array_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 4); - EXPECT_EQ(offsets[4], 5); - - const auto& elements = - assert_cast(array_column.get_data()); - const auto& struct_column = - assert_cast(elements.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_FALSE(elements.is_null_at(0)); - EXPECT_FALSE(elements.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_FALSE(elements.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 11); - EXPECT_EQ(a_values.get_element(1), 12); - EXPECT_EQ(a_values.get_element(3), 13); - EXPECT_EQ(a_values.get_element(4), 14); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); - }); - auto nested_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field(arrow::field("nullable_list_list_int_col", - arrow::list(arrow::field("element", nested_list_type, true)), true), - build_nullable_list_list_int_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 4); - EXPECT_EQ(outer_offsets[3], 5); - EXPECT_EQ(outer_offsets[4], 7); - - const auto& inner_nullable = - assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_FALSE(inner_nullable.is_null_at(6)); - - const auto& inner_array = - assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[5], 5); - EXPECT_EQ(inner_offsets[6], 7); - - const auto& elements = - assert_cast(inner_array.get_data()); - const auto& values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 7); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 3); - EXPECT_EQ(values.get_element(4), 4); - EXPECT_EQ(values.get_element(5), 5); - EXPECT_TRUE(elements.is_null_at(6)); - }); - add_field(arrow::field( - "map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), false)), - false), - build_required_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_MAP); - const auto* map_type = - assert_cast(remove_nullable(schema.type).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), - TYPE_INT); - EXPECT_EQ(remove_nullable(map_type->get_value_type())->get_primitive_type(), - TYPE_STRING); - const auto& map_column = assert_cast(column); - ASSERT_EQ(map_column.size(), ROW_COUNT); - const auto map_size_at = [&map_column](size_t row_idx) { - return map_column.get_offsets()[row_idx] - - (row_idx == 0 ? 0 : map_column.get_offsets()[row_idx - 1]); - }; - EXPECT_EQ(map_size_at(0), 2); - EXPECT_EQ(map_size_at(1), 1); - EXPECT_EQ(map_size_at(2), 3); - EXPECT_EQ(map_size_at(4), 2); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(keys.size(), 9); - ASSERT_EQ(values.size(), 9); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(5), 6); - EXPECT_EQ(keys.get_element(8), 9); - EXPECT_EQ(values.get_data_at(0).to_string(), "a"); - EXPECT_EQ(values.get_data_at(5).to_string(), "f"); - EXPECT_EQ(values.get_data_at(8).to_string(), "i"); - }); - add_field( - arrow::field("nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - true), - build_nullable_int_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 10); - EXPECT_EQ(keys.get_element(1), 20); - EXPECT_EQ(keys.get_element(3), 40); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(values.is_null_at(3)); - }); - add_field( - arrow::field("required_nullable_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)), - false), - build_required_nullable_string_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_FALSE(schema.type->is_nullable()); - const auto& map_column = assert_cast(column); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 4); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(3)); - }); - auto map_struct_type = arrow::struct_({ - arrow::field("a", arrow::int32(), false), - arrow::field("b", arrow::utf8(), true), - }); - add_field(arrow::field( - "nullable_map_int_struct_col", - arrow::map(arrow::int32(), arrow::field("value", map_struct_type, true)), - true), - build_nullable_int_struct_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 3); - EXPECT_EQ(offsets[4], 4); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& struct_column = - assert_cast(values.get_nested_column()); - const auto& a_values = - get_nullable_nested_column(struct_column.get_column(0)); - const auto& b_values = - assert_cast(struct_column.get_column(1)); - const auto& b_data = - assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(a_values.get_element(0), 21); - EXPECT_EQ(a_values.get_element(1), 22); - EXPECT_EQ(a_values.get_element(3), 24); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); - }); - auto map_list_type = arrow::list(arrow::field("element", arrow::int32(), true)); - add_field( - arrow::field("nullable_map_int_list_col", - arrow::map(arrow::int32(), arrow::field("value", map_list_type, true)), - true), - build_nullable_int_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& map_column = - assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), ROW_COUNT); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 4); - EXPECT_EQ(map_offsets[4], 5); - - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 5); - EXPECT_EQ(keys.get_element(0), 201); - EXPECT_EQ(keys.get_element(1), 202); - EXPECT_EQ(keys.get_element(2), 203); - EXPECT_EQ(keys.get_element(3), 204); - EXPECT_EQ(keys.get_element(4), 205); - - const auto& values = - assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_FALSE(values.is_null_at(4)); - - const auto& list_column = - assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); - - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(element_values.get_element(0), 1); - EXPECT_EQ(element_values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(element_values.get_element(3), 3); - EXPECT_EQ(element_values.get_element(4), 4); - }); - auto list_map_type = arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field("nullable_list_map_int_string_col", - arrow::list(arrow::field("element", list_map_type, true)), true), - build_nullable_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& map_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 3); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - const auto& values = - assert_cast(map_column.get_values()); - const auto& value_data = - assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(value_data.get_data_at(3).to_string(), "d"); - }); - auto nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - add_field(arrow::field( - "nullable_map_int_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", nested_map_type, true)), - true), - build_nullable_int_map_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 4); - - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 4); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - - const auto& inner_values = - assert_cast(outer_map.get_values()); - ASSERT_EQ(inner_values.size(), 4); - EXPECT_FALSE(inner_values.is_null_at(0)); - EXPECT_FALSE(inner_values.is_null_at(1)); - EXPECT_TRUE(inner_values.is_null_at(2)); - EXPECT_FALSE(inner_values.is_null_at(3)); - - const auto& inner_map = - assert_cast(inner_values.get_nested_column()); - const auto& inner_offsets = inner_map.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 4); - EXPECT_EQ(inner_offsets[0], 1); - EXPECT_EQ(inner_offsets[1], 1); - EXPECT_EQ(inner_offsets[2], 1); - EXPECT_EQ(inner_offsets[3], 2); - const auto& inner_keys = - get_nullable_nested_column(inner_map.get_keys()); - const auto& inner_strings = - assert_cast(inner_map.get_values()); - const auto& inner_string_data = - assert_cast(inner_strings.get_nested_column()); - ASSERT_EQ(inner_keys.size(), 2); - EXPECT_EQ(inner_keys.get_element(0), 101); - EXPECT_EQ(inner_keys.get_element(1), 401); - EXPECT_EQ(inner_string_data.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(inner_strings.is_null_at(1)); - }); - auto deep_list_value_type = arrow::list(arrow::field("element", arrow::int32(), true)); - auto deep_list_map_type = - arrow::map(arrow::int32(), arrow::field("value", deep_list_value_type, true)); - auto deep_list_struct_type = arrow::struct_({arrow::field("kv", deep_list_map_type, true)}); - add_field(arrow::field("nullable_list_struct_map_list_col", - arrow::list(arrow::field("element", deep_list_struct_type, true)), - true), - build_deep_list_struct_map_list_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_array = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - - const auto& struct_values = - assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - - const auto& struct_column = - assert_cast(struct_values.get_nested_column()); - const auto& map_values = - assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - - const auto& map_column = - assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); - const auto& keys = - get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 4); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 3); - EXPECT_EQ(keys.get_element(3), 4); - - const auto& lists = - assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 4); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 4); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 3); - const auto& elements = - assert_cast(list_column.get_data()); - const auto& element_values = - assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 3); - EXPECT_EQ(element_values.get_element(0), 10); - EXPECT_TRUE(elements.is_null_at(1)); - EXPECT_EQ(element_values.get_element(2), 40); - }); - auto deep_map_nested_map_type = - arrow::map(arrow::int32(), arrow::field("value", arrow::utf8(), true)); - auto deep_map_list_type = - arrow::list(arrow::field("element", deep_map_nested_map_type, true)); - add_field( - arrow::field( - "nullable_map_int_list_map_int_string_col", - arrow::map(arrow::int32(), arrow::field("value", deep_map_list_type, true)), - true), - build_deep_map_list_map_array(), - [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& outer_map = - assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), ROW_COUNT); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 4); - EXPECT_EQ(outer_offsets[4], 5); - const auto& outer_keys = - get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = - assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); - - const auto& inner_maps = - assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 6); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_FALSE(inner_maps.is_null_at(1)); - EXPECT_TRUE(inner_maps.is_null_at(2)); - EXPECT_FALSE(inner_maps.is_null_at(3)); - EXPECT_TRUE(inner_maps.is_null_at(4)); - EXPECT_FALSE(inner_maps.is_null_at(5)); - const auto& inner_map_column = - assert_cast(inner_maps.get_nested_column()); - const auto& inner_offsets = inner_map_column.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 6); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 2); - EXPECT_EQ(inner_offsets[2], 2); - EXPECT_EQ(inner_offsets[3], 3); - EXPECT_EQ(inner_offsets[4], 3); - EXPECT_EQ(inner_offsets[5], 4); - const auto& inner_keys = - get_nullable_nested_column(inner_map_column.get_keys()); - ASSERT_EQ(inner_keys.size(), 4); - EXPECT_EQ(inner_keys.get_element(0), 1); - EXPECT_EQ(inner_keys.get_element(1), 2); - EXPECT_EQ(inner_keys.get_element(2), 3); - EXPECT_EQ(inner_keys.get_element(3), 4); - const auto& strings = - assert_cast(inner_map_column.get_values()); - const auto& string_data = - assert_cast(strings.get_nested_column()); - ASSERT_EQ(strings.size(), 4); - EXPECT_EQ(string_data.get_data_at(0).to_string(), "a"); - EXPECT_TRUE(strings.is_null_at(1)); - EXPECT_EQ(string_data.get_data_at(2).to_string(), "c"); - EXPECT_EQ(string_data.get_data_at(3).to_string(), "d"); - }); - - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - - auto file_result = arrow::io::FileOutputStream::Open(_file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ROW_COUNT, builder.build())); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_plain_reader(size_t field_idx) { - // Keep the normal fixture dictionary encoded. This one test writes a plain-encoded copy - // because Arrow BinaryRecordReader has a stricter reset contract than - // DictionaryRecordReader. - auto schema = arrow::schema(_arrow_fields); - auto table = arrow::Table::Make(schema, _arrays); - auto plain_file_result = arrow::io::FileOutputStream::Open(_plain_file_path); - DORIS_CHECK(plain_file_result.ok()); - std::shared_ptr plain_out = *plain_file_result; - ::parquet::WriterProperties::Builder plain_builder; - plain_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - plain_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - plain_builder.compression(::parquet::Compression::UNCOMPRESSED); - plain_builder.disable_dictionary(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), plain_out, ROW_COUNT, plain_builder.build())); - DORIS_CHECK(plain_out->Close().ok()); - - _plain_file_reader = ::parquet::ParquetFileReader::OpenFile(_plain_file_path, false); - auto metadata = _plain_file_reader->metadata(); - DORIS_CHECK(metadata != nullptr); - DORIS_CHECK(metadata->num_row_groups() == 1); - _plain_row_group = _plain_file_reader->RowGroup(0); - DORIS_CHECK(_plain_row_group != nullptr); - - ParquetColumnReaderFactory factory(_plain_row_group, metadata->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_child_reader(size_t field_idx, - size_t child_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[child_idx]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - std::unique_ptr create_projected_grandchild_reader( - size_t field_idx, size_t child_idx, size_t grandchild_idx) const { - const auto& struct_schema = *_fields[field_idx]; - EXPECT_LT(child_idx, struct_schema.children.size()); - const auto& child_schema = *struct_schema.children[child_idx]; - EXPECT_LT(grandchild_idx, child_schema.children.size()); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = child_schema.local_id; - child_projection.project_all_children = false; - format::LocalColumnIndex grandchild_projection; - grandchild_projection.index = child_schema.children[grandchild_idx]->local_id; - child_projection.children.push_back(std::move(grandchild_projection)); - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - void read_and_validate(size_t field_idx) const { - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - _expected_by_field[field_idx](*_fields[field_idx], *column); - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet test field " << name; - return _fields.size(); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::string _plain_file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::unique_ptr<::parquet::ParquetFileReader> _plain_file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::shared_ptr<::parquet::RowGroupReader> _plain_row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; - std::vector> _expected_by_field; -}; - -TEST(ParquetColumnReaderBaseTest, SelectionVectorRangesAndValidation) { - SelectionVector identity; - ASSERT_TRUE(identity.verify(4, 5).ok()); - auto ranges = selection_to_ranges(identity, 4); - ASSERT_EQ(ranges.size(), 1); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 4); - - std::array selected = {0, 2, 3, 6, 6}; - SelectionVector external(selected.data(), 4); - auto status = external.verify(3, 7); - ASSERT_TRUE(status.ok()) << status; - ranges = selection_to_ranges(external, 3); - ASSERT_EQ(ranges.size(), 2); - EXPECT_EQ(ranges[0].start, 0); - EXPECT_EQ(ranges[0].length, 1); - EXPECT_EQ(ranges[1].start, 2); - EXPECT_EQ(ranges[1].length, 2); - - EXPECT_FALSE(external.verify(8, 7).ok()); - EXPECT_FALSE(external.verify(5, 7).ok()); - EXPECT_FALSE(external.verify(4, 6).ok()); - - std::array duplicate = {0, 2, 2}; - SelectionVector non_strict(duplicate.data(), duplicate.size()); - EXPECT_FALSE(non_strict.verify(3, 5).ok()); - EXPECT_FALSE(identity.verify(1, -1).ok()); -} - -TEST(ParquetColumnReaderBaseTest, DefaultSelectUsesSkipReadRangesAndNestedConsumeIsExplicit) { - DefaultSelectReader reader; - std::array selected = {1, 3, 4}; - SelectionVector selection(selected.data(), selected.size()); - auto column = ColumnInt32::create(); - MutableColumnPtr mutable_column = std::move(column); - auto status = reader.select(selection, selected.size(), 6, mutable_column); - ASSERT_TRUE(status.ok()) << status; - - const auto& values = assert_cast(*mutable_column); - ASSERT_EQ(values.size(), 3); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 3); - EXPECT_EQ(values.get_element(2), 4); - EXPECT_EQ(reader.skip_ranges(), std::vector({1, 1, 1})); - EXPECT_EQ(reader.read_ranges(), std::vector({1, 2})); - - BaseUnsupportedReader unsupported_reader; - auto skip_status = unsupported_reader.skip(1); - EXPECT_FALSE(skip_status.ok()); - EXPECT_NE(skip_status.to_string().find("skip is not implemented"), std::string::npos); - EXPECT_FALSE(unsupported_reader.load_nested_batch(1).ok()); - int64_t values_read = 0; - EXPECT_FALSE(unsupported_reader.build_nested_column(1, mutable_column, &values_read).ok()); - EXPECT_FALSE(unsupported_reader.consume_nested_column(1, &values_read).ok()); - - NestedSkipReader nested_reader; - auto nested_status = nested_reader.consume_nested_column(3, &values_read); - ASSERT_TRUE(nested_status.ok()) << nested_status; - EXPECT_EQ(values_read, 3); -} - -TEST_F(ParquetColumnReaderTest, ScalarReadCoversRequiredNullableAllNullAndMultipleBatches) { - read_and_validate(find_field_idx("int32_col")); - read_and_validate(find_field_idx("string_col")); - read_and_validate(find_field_idx("nullable_int_col")); - read_and_validate(find_field_idx("all_null_int_col")); - - auto reader = create_reader(find_field_idx("int32_col")); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - ASSERT_TRUE(reader->read(3, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 3); - const auto& values = assert_cast(*column); - ASSERT_EQ(values.size(), ROW_COUNT); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(1), 20); - EXPECT_EQ(values.get_element(2), 30); - EXPECT_EQ(values.get_element(4), 50); -} - -TEST_F(ParquetColumnReaderTest, ScalarSkipCoversZeroSomeAllAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(0).ok()); - auto column = reader->type()->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& first_value = assert_cast(*column); - EXPECT_EQ(first_value.get_element(0), 10); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(2).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& skipped_values = assert_cast(*column); - EXPECT_EQ(skipped_values.get_element(0), 30); - EXPECT_EQ(skipped_values.get_element(1), 40); - - reader = create_reader(find_field_idx("int32_col")); - ASSERT_TRUE(reader->skip(ROW_COUNT).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - EXPECT_EQ(rows_read, 0); - EXPECT_EQ(column->size(), 0); - - reader = create_reader(find_field_idx("nullable_int_col")); - ASSERT_TRUE(reader->skip(1).ok()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->read(2, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 2); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); -} - -TEST_F(ParquetColumnReaderTest, ScalarSelectCoversAllDisjointSingleZeroThenReadAndNulls) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector all_selected(ROW_COUNT); - auto column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(all_selected, ROW_COUNT, ROW_COUNT, column).ok()); - const auto& all_values = assert_cast(*column); - ASSERT_EQ(all_values.size(), ROW_COUNT); - EXPECT_EQ(all_values.get_element(0), 10); - EXPECT_EQ(all_values.get_element(4), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array disjoint = {0, 2, 4}; - SelectionVector disjoint_selection(disjoint.data(), disjoint.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(disjoint_selection, disjoint.size(), ROW_COUNT, column).ok()); - const auto& disjoint_values = assert_cast(*column); - ASSERT_EQ(disjoint_values.size(), 3); - EXPECT_EQ(disjoint_values.get_element(0), 10); - EXPECT_EQ(disjoint_values.get_element(1), 30); - EXPECT_EQ(disjoint_values.get_element(2), 50); - - reader = create_reader(find_field_idx("int32_col")); - std::array single = {2}; - SelectionVector single_selection(single.data(), single.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(single_selection, single.size(), ROW_COUNT, column).ok()); - const auto& single_value = assert_cast(*column); - ASSERT_EQ(single_value.size(), 1); - EXPECT_EQ(single_value.get_element(0), 30); - - reader = create_reader(find_field_idx("int32_col")); - std::array first_last = {0, 4}; - SelectionVector first_last_selection(first_last.data(), first_last.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(first_last_selection, first_last.size(), ROW_COUNT, column).ok()); - const auto& first_last_values = assert_cast(*column); - ASSERT_EQ(first_last_values.size(), 2); - EXPECT_EQ(first_last_values.get_element(0), 10); - EXPECT_EQ(first_last_values.get_element(1), 50); - - reader = create_reader(find_field_idx("int32_col")); - SelectionVector empty_selection; - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(empty_selection, 0, 2, column).ok()); - ASSERT_EQ(column->size(), 0); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(1, column, &rows_read).ok()); - ASSERT_EQ(rows_read, 1); - const auto& after_empty_select = assert_cast(*column); - ASSERT_EQ(after_empty_select.size(), 1); - EXPECT_EQ(after_empty_select.get_element(0), 30); - - reader = create_reader(find_field_idx("nullable_int_col")); - std::array nullable_rows = {0, 1, 2}; - SelectionVector nullable_selection(nullable_rows.data(), nullable_rows.size()); - column = reader->type()->create_column(); - ASSERT_TRUE(reader->select(nullable_selection, nullable_rows.size(), ROW_COUNT, column).ok()); - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidScalarInputsAndNestedScalarProjection) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& int_schema = *_fields[find_field_idx("int32_col")]; - ParquetColumnSchema invalid_leaf; - invalid_leaf.kind = ParquetColumnSchemaKind::PRIMITIVE; - invalid_leaf.name = "invalid_leaf"; - invalid_leaf.type = int_schema.type; - invalid_leaf.type_descriptor = int_schema.type_descriptor; - invalid_leaf.descriptor = int_schema.descriptor; - invalid_leaf.leaf_column_id = _file_reader->metadata()->num_columns(); - auto status = factory.create(invalid_leaf, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); - - ParquetColumnSchema null_descriptor; - null_descriptor.kind = ParquetColumnSchemaKind::PRIMITIVE; - null_descriptor.name = "null_descriptor"; - null_descriptor.type = int_schema.type; - null_descriptor.type_descriptor = int_schema.type_descriptor; - null_descriptor.leaf_column_id = int_schema.leaf_column_id; - status = factory.create(null_descriptor, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("descriptor is null"), std::string::npos); - - const auto& list_element_schema = - *_fields[find_field_idx("nullable_list_int_col")]->children[0]; - status = factory.create(list_element_schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("flat primitive columns"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex projection = - format::LocalColumnIndex::partial_local(list_schema.local_id); - format::LocalColumnIndex element_projection = - format::LocalColumnIndex::partial_local(list_element_schema.local_id); - projection.children.push_back(std::move(element_projection)); - status = factory.create(list_schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("scalar projection is invalid"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, FactoryRejectsInvalidComplexProjections) { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - - const auto& struct_schema = *_fields[find_field_idx("struct_col")]; - format::LocalColumnIndex struct_empty = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - auto status = factory.create(struct_schema, &struct_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); - - format::LocalColumnIndex struct_invalid = - format::LocalColumnIndex::partial_local(struct_schema.local_id); - struct_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(struct_schema, &struct_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); - - const auto& list_schema = *_fields[find_field_idx("nullable_list_int_col")]; - format::LocalColumnIndex list_empty = - format::LocalColumnIndex::partial_local(list_schema.local_id); - status = factory.create(list_schema, &list_empty, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); - - const auto& map_schema = *_fields[find_field_idx("nullable_map_int_struct_col")]; - const auto& value_schema = *map_schema.children[1]; - format::LocalColumnIndex map_invalid = - format::LocalColumnIndex::partial_local(map_schema.local_id); - map_invalid.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - map_invalid.children.push_back(format::LocalColumnIndex::local(9999)); - status = factory.create(map_schema, &map_invalid, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains invalid child"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadSupportedComplexTypes) { - read_and_validate(find_field_idx("struct_col")); - read_and_validate(find_field_idx("nullable_struct_col")); - read_and_validate(find_field_idx("nullable_struct_decimal_col")); - read_and_validate(find_field_idx("list_int_col")); - read_and_validate(find_field_idx("nullable_list_int_col")); - read_and_validate(find_field_idx("required_nullable_list_int_col")); - read_and_validate(find_field_idx("nullable_list_struct_col")); - read_and_validate(find_field_idx("nullable_list_list_int_col")); - read_and_validate(find_field_idx("map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_string_col")); - read_and_validate(find_field_idx("required_nullable_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_struct_col")); - read_and_validate(find_field_idx("nullable_map_int_list_col")); - read_and_validate(find_field_idx("nullable_list_map_int_string_col")); - read_and_validate(find_field_idx("nullable_map_int_map_int_string_col")); - read_and_validate(find_field_idx("nullable_list_struct_map_list_col")); - read_and_validate(find_field_idx("nullable_map_int_list_map_int_string_col")); -} - -TEST_F(ParquetColumnReaderTest, SkipThenRead) { - auto reader = create_reader(find_field_idx("int32_col")); - auto st = reader->skip(2); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 2); - EXPECT_EQ(int_values.get_element(0), 30); - EXPECT_EQ(int_values.get_element(1), 40); -} - -TEST_F(ParquetColumnReaderTest, SelectReadsOnlySelectedRanges) { - auto reader = create_reader(find_field_idx("int32_col")); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 2); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& int_values = assert_cast(*column); - ASSERT_EQ(int_values.size(), 3); - EXPECT_EQ(int_values.get_element(0), 10); - EXPECT_EQ(int_values.get_element(1), 30); - EXPECT_EQ(int_values.get_element(2), 50); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructChildren) { - const auto field_idx = find_field_idx("struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& struct_column = assert_cast(*column); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(values.get_data_at(0).to_string(), "sa"); - EXPECT_EQ(values.get_data_at(4).to_string(), "se"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedNullableStructChildren) { - const auto field_idx = find_field_idx("nullable_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = struct_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = struct_schema.children[1]->local_id; - projection.children.push_back(std::move(child_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(struct_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - ASSERT_EQ(remove_nullable(reader->type())->get_primitive_type(), TYPE_STRUCT); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - const auto& nullable_column = assert_cast(*column); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_TRUE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& values = assert_cast(struct_column.get_column(0)); - const auto& nested_values = assert_cast(values.get_nested_column()); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(3)); - EXPECT_EQ(nested_values.get_data_at(0).to_string(), "nsa"); - EXPECT_EQ(nested_values.get_data_at(3).to_string(), "nsd"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedListStructElementChildren) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& list_schema = *_fields[field_idx]; - ASSERT_EQ(list_schema.name, "nullable_list_struct_col"); - ASSERT_EQ(list_schema.children.size(), 1); - const auto& element_schema = *list_schema.children[0]; - ASSERT_EQ(element_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = list_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex element_projection; - element_projection.index = element_schema.local_id; - element_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = element_schema.children[1]->local_id; - element_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(element_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(list_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* array_type = - assert_cast(remove_nullable(reader->type()).get()); - const auto* element_type = assert_cast( - remove_nullable(array_type->get_nested_type()).get()); - ASSERT_EQ(element_type->get_elements().size(), 1); - EXPECT_EQ(element_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& elements = assert_cast(array_column.get_data()); - const auto& struct_column = assert_cast(elements.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(elements.size(), 5); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "la"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "ld"); - EXPECT_EQ(b_data.get_data_at(4).to_string(), "le"); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedMapStructValueChildren) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.name, "nullable_map_int_struct_col"); - ASSERT_EQ(map_schema.children.size(), 2); - const auto& value_schema = *map_schema.children[1]; - ASSERT_EQ(value_schema.children.size(), 2); - - format::LocalColumnIndex projection; - projection.index = map_schema.local_id; - projection.project_all_children = false; - format::LocalColumnIndex value_projection; - value_projection.index = value_schema.local_id; - value_projection.project_all_children = false; - format::LocalColumnIndex child_projection; - child_projection.index = value_schema.children[1]->local_id; - value_projection.children.push_back(std::move(child_projection)); - projection.children.push_back(std::move(value_projection)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* map_type = assert_cast(remove_nullable(reader->type()).get()); - EXPECT_EQ(remove_nullable(map_type->get_key_type())->get_primitive_type(), TYPE_INT); - const auto* value_type = - assert_cast(remove_nullable(map_type->get_value_type()).get()); - ASSERT_EQ(value_type->get_elements().size(), 1); - EXPECT_EQ(value_type->get_element_name(0), "b"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - const auto& values = assert_cast(map_column.get_values()); - const auto& struct_column = assert_cast(values.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& b_values = assert_cast(struct_column.get_column(0)); - const auto& b_data = assert_cast(b_values.get_nested_column()); - ASSERT_EQ(keys.size(), 4); - ASSERT_EQ(values.size(), 4); - EXPECT_EQ(keys.get_element(0), 101); - EXPECT_EQ(keys.get_element(1), 102); - EXPECT_EQ(keys.get_element(3), 104); - EXPECT_EQ(b_data.get_data_at(0).to_string(), "ma"); - EXPECT_TRUE(b_values.is_null_at(1)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_EQ(b_data.get_data_at(3).to_string(), "me"); -} - -TEST_F(ParquetColumnReaderTest, AllowsMapKeyWithValueProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - const auto& value_schema = *map_schema.children[1]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - projection.children.push_back(format::LocalColumnIndex::local(value_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_TRUE(st.ok()) << st; - ASSERT_NE(reader, nullptr); -} - -TEST_F(ParquetColumnReaderTest, RejectMapKeyOnlyProjection) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& map_schema = *_fields[field_idx]; - ASSERT_EQ(map_schema.children.size(), 2); - const auto& key_schema = *map_schema.children[0]; - - auto projection = format::LocalColumnIndex::partial_local(map_schema.local_id); - projection.children.push_back(format::LocalColumnIndex::local(key_schema.local_id)); - - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - const auto st = factory.create(map_schema, &projection, &reader); - ASSERT_FALSE(st.ok()); - EXPECT_NE(st.to_string().find("contains no value"), std::string::npos); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_list_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "xs"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), ROW_COUNT); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - EXPECT_TRUE(xs_nullable.is_null_at(3)); - EXPECT_FALSE(xs_nullable.is_null_at(4)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 4); - const auto& elements = assert_cast(xs_array.get_data()); - const auto& values = assert_cast(elements.get_nested_column()); - ASSERT_EQ(elements.size(), 4); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 2); - EXPECT_TRUE(elements.is_null_at(2)); - EXPECT_EQ(values.get_element(3), 5); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructListChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructListChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& xs_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - ASSERT_LT(field_idx, _fields.size()); - const auto& struct_schema = *_fields[field_idx]; - ASSERT_EQ(struct_schema.name, "nullable_struct_map_col"); - ASSERT_EQ(struct_schema.children.size(), 2); - - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - ASSERT_TRUE(reader->type()->is_nullable()); - const auto* projected_type = - assert_cast(remove_nullable(reader->type()).get()); - ASSERT_EQ(projected_type->get_elements().size(), 1); - EXPECT_EQ(projected_type->get_element_name(0), "kv"); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), ROW_COUNT); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - EXPECT_TRUE(kv_nullable.is_null_at(3)); - EXPECT_FALSE(kv_nullable.is_null_at(4)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), ROW_COUNT); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 2); - EXPECT_EQ(offsets[3], 2); - EXPECT_EQ(offsets[4], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesListChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesMapChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, NullableStructUsesNestedStructComplexChildAsShapeSource) { - const auto field_idx = find_field_idx("nullable_struct_nested_struct_list_col"); - auto reader = create_projected_grandchild_reader(field_idx, 0, 0); - ASSERT_NE(reader, nullptr); - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_FALSE(nullable_column.is_null_at(4)); - - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& nested_nullable = assert_cast(struct_column.get_column(0)); - EXPECT_FALSE(nested_nullable.is_null_at(0)); - EXPECT_TRUE(nested_nullable.is_null_at(2)); - EXPECT_FALSE(nested_nullable.is_null_at(3)); - EXPECT_FALSE(nested_nullable.is_null_at(4)); -} - -TEST_F(ParquetColumnReaderTest, SkipProjectedStructMapChildOnlyThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectProjectedStructMapChildOnly) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_projected_child_reader(field_idx, 1); - ASSERT_NE(reader, nullptr); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(struct_column.get_columns().size(), 1); - const auto& kv_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadStructListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(1)); - EXPECT_TRUE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructListWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 301); - EXPECT_EQ(a_values.get_element(1), 304); - EXPECT_EQ(a_values.get_element(2), 305); - const auto& xs_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(xs_nullable.size(), 3); - EXPECT_FALSE(xs_nullable.is_null_at(0)); - EXPECT_TRUE(xs_nullable.is_null_at(1)); - EXPECT_FALSE(xs_nullable.is_null_at(2)); - const auto& xs_array = assert_cast(xs_nullable.get_nested_column()); - const auto& offsets = xs_array.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadStructMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipStructMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(1)); - EXPECT_TRUE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 0); -} - -TEST_F(ParquetColumnReaderTest, SelectStructMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_struct_map_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& struct_column = - assert_cast(nullable_column.get_nested_column()); - const auto& a_values = get_nullable_nested_column(struct_column.get_column(0)); - EXPECT_EQ(a_values.get_element(0), 401); - EXPECT_EQ(a_values.get_element(1), 404); - EXPECT_EQ(a_values.get_element(2), 405); - const auto& kv_nullable = assert_cast(struct_column.get_column(1)); - ASSERT_EQ(kv_nullable.size(), 3); - EXPECT_FALSE(kv_nullable.is_null_at(0)); - EXPECT_TRUE(kv_nullable.is_null_at(1)); - EXPECT_FALSE(kv_nullable.is_null_at(2)); - const auto& kv_map = assert_cast(kv_nullable.get_nested_column()); - const auto& offsets = kv_map.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 2); - EXPECT_EQ(offsets[2], 3); - const auto& keys = get_nullable_nested_column(kv_map.get_keys()); - const auto& values = assert_cast(kv_map.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(keys.size(), 3); - EXPECT_EQ(keys.get_element(0), 1); - EXPECT_EQ(keys.get_element(1), 2); - EXPECT_EQ(keys.get_element(2), 5); - EXPECT_EQ(value_data.get_data_at(0).to_string(), "one"); - EXPECT_TRUE(values.is_null_at(1)); - EXPECT_EQ(value_data.get_data_at(2).to_string(), "five"); -} - -TEST_F(ParquetColumnReaderTest, ReadListStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectListStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& array_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = array_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 4); - EXPECT_EQ(offsets[2], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadListListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipListListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 1); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 1); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 1); - EXPECT_EQ(inner_offsets[0], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectListListWithOverflow) { - const auto field_idx = find_field_idx("nullable_list_list_int_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 4); - EXPECT_EQ(outer_offsets[1], 5); - EXPECT_EQ(outer_offsets[2], 7); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 7); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 7); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 5); - EXPECT_EQ(inner_offsets[6], 7); -} - -TEST_F(ParquetColumnReaderTest, ReadMapWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SkipPlainBinaryMapThenReadResetsArrowBuilder) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_plain_reader(field_idx); - - // Row 0 contains two STRING values. The levels-only skip must release (and discard) those - // Arrow BinaryRecordReader builder chunks before the next normal read. If they leak into the - // next batch, ParquetLeafReader observes more values than current definition/repetition levels. - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 3); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(map_column.get_offsets()[1], 0); - EXPECT_EQ(map_column.get_offsets()[2], 1); - const auto& values = get_nullable_nested_column(map_column.get_values()); - ASSERT_EQ(values.size(), 1); - EXPECT_EQ(values.get_data_at(0).to_string(), "cc"); -} - -TEST_F(ParquetColumnReaderTest, SelectMapWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapStructWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapStructWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 0); - EXPECT_EQ(offsets[1], 0); - EXPECT_EQ(offsets[2], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectMapStructWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_struct_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& offsets = map_column.get_offsets(); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 2); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadMapListWithOverflowAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipMapListWithOverflowThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(3, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 3); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_TRUE(nullable_column.is_null_at(0)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 2); - EXPECT_TRUE(values.is_null_at(0)); - EXPECT_FALSE(values.is_null_at(1)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 2); -} - -TEST_F(ParquetColumnReaderTest, SelectMapListWithOverflow) { - const auto field_idx = find_field_idx("nullable_map_int_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& map_column = assert_cast(nullable_column.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 4); - EXPECT_EQ(map_offsets[2], 5); - - const auto& values = assert_cast(map_column.get_values()); - ASSERT_EQ(values.size(), 5); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_TRUE(values.is_null_at(2)); - EXPECT_FALSE(values.is_null_at(4)); - const auto& list_column = assert_cast(values.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 2); - EXPECT_EQ(list_offsets[1], 2); - EXPECT_EQ(list_offsets[2], 2); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 5); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepListStructMapListAcrossChunks) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepListStructMapListThenRead) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 3); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_FALSE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 3); - EXPECT_TRUE(map_values.is_null_at(0)); - EXPECT_FALSE(map_values.is_null_at(1)); - EXPECT_FALSE(map_values.is_null_at(2)); - - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 3); - EXPECT_EQ(map_offsets[0], 0); - EXPECT_EQ(map_offsets[1], 0); - EXPECT_EQ(map_offsets[2], 2); - const auto& keys = get_nullable_nested_column(map_column.get_keys()); - ASSERT_EQ(keys.size(), 2); - EXPECT_EQ(keys.get_element(0), 3); - EXPECT_EQ(keys.get_element(1), 4); - const auto& lists = assert_cast(map_column.get_values()); - ASSERT_EQ(lists.size(), 2); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 2); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepListStructMapList) { - const auto field_idx = find_field_idx("nullable_list_struct_map_list_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - - const auto& struct_values = assert_cast(outer_array.get_data()); - ASSERT_EQ(struct_values.size(), 5); - EXPECT_FALSE(struct_values.is_null_at(0)); - EXPECT_TRUE(struct_values.is_null_at(1)); - EXPECT_FALSE(struct_values.is_null_at(2)); - EXPECT_FALSE(struct_values.is_null_at(3)); - EXPECT_FALSE(struct_values.is_null_at(4)); - const auto& struct_column = assert_cast(struct_values.get_nested_column()); - const auto& map_values = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_values.size(), 5); - EXPECT_FALSE(map_values.is_null_at(0)); - EXPECT_TRUE(map_values.is_null_at(1)); - EXPECT_TRUE(map_values.is_null_at(2)); - EXPECT_FALSE(map_values.is_null_at(3)); - EXPECT_FALSE(map_values.is_null_at(4)); - const auto& map_column = assert_cast(map_values.get_nested_column()); - const auto& map_offsets = map_column.get_offsets(); - ASSERT_EQ(map_offsets.size(), 5); - EXPECT_EQ(map_offsets[0], 2); - EXPECT_EQ(map_offsets[1], 2); - EXPECT_EQ(map_offsets[2], 2); - EXPECT_EQ(map_offsets[3], 2); - EXPECT_EQ(map_offsets[4], 4); -} - -TEST_F(ParquetColumnReaderTest, ReadDeepMapListMapAcrossChunks) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - MutableColumnPtr column = reader->type()->create_column(); - - int64_t rows_read = 0; - auto st = reader->read(1, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 1); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - st = reader->read(2, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 2); - - _expected_by_field[field_idx](*_fields[field_idx], *column); -} - -TEST_F(ParquetColumnReaderTest, SkipDeepMapListMapThenRead) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - auto st = reader->skip(1); - ASSERT_TRUE(st.ok()) << st; - - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - st = reader->read(4, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 0); - EXPECT_EQ(outer_offsets[2], 2); - EXPECT_EQ(outer_offsets[3], 3); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 3); - EXPECT_EQ(outer_keys.get_element(0), 30); - EXPECT_EQ(outer_keys.get_element(1), 40); - EXPECT_EQ(outer_keys.get_element(2), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 3); - EXPECT_TRUE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_FALSE(lists.is_null_at(2)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 3); - EXPECT_EQ(list_offsets[0], 0); - EXPECT_EQ(list_offsets[1], 1); - EXPECT_EQ(list_offsets[2], 3); - const auto& inner_maps = assert_cast(list_column.get_data()); - ASSERT_EQ(inner_maps.size(), 3); - EXPECT_FALSE(inner_maps.is_null_at(0)); - EXPECT_TRUE(inner_maps.is_null_at(1)); - EXPECT_FALSE(inner_maps.is_null_at(2)); -} - -TEST_F(ParquetColumnReaderTest, SelectDeepMapListMap) { - const auto field_idx = find_field_idx("nullable_map_int_list_map_int_string_col"); - auto reader = create_reader(field_idx); - SelectionVector selection(3); - selection.set_index(0, 0); - selection.set_index(1, 3); - selection.set_index(2, 4); - - MutableColumnPtr column = reader->type()->create_column(); - auto st = reader->select(selection, 3, ROW_COUNT, column); - ASSERT_TRUE(st.ok()) << st; - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 3); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - const auto& outer_map = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_map.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 3); - EXPECT_EQ(outer_offsets[0], 2); - EXPECT_EQ(outer_offsets[1], 4); - EXPECT_EQ(outer_offsets[2], 5); - const auto& outer_keys = get_nullable_nested_column(outer_map.get_keys()); - ASSERT_EQ(outer_keys.size(), 5); - EXPECT_EQ(outer_keys.get_element(0), 10); - EXPECT_EQ(outer_keys.get_element(1), 20); - EXPECT_EQ(outer_keys.get_element(2), 30); - EXPECT_EQ(outer_keys.get_element(3), 40); - EXPECT_EQ(outer_keys.get_element(4), 50); - - const auto& lists = assert_cast(outer_map.get_values()); - ASSERT_EQ(lists.size(), 5); - EXPECT_FALSE(lists.is_null_at(0)); - EXPECT_FALSE(lists.is_null_at(1)); - EXPECT_TRUE(lists.is_null_at(2)); - EXPECT_FALSE(lists.is_null_at(3)); - EXPECT_FALSE(lists.is_null_at(4)); - const auto& list_column = assert_cast(lists.get_nested_column()); - const auto& list_offsets = list_column.get_offsets(); - ASSERT_EQ(list_offsets.size(), 5); - EXPECT_EQ(list_offsets[0], 3); - EXPECT_EQ(list_offsets[1], 3); - EXPECT_EQ(list_offsets[2], 3); - EXPECT_EQ(list_offsets[3], 4); - EXPECT_EQ(list_offsets[4], 6); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp b/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp deleted file mode 100644 index 438aac91847bbd..00000000000000 --- a/be/test/format_v2/parquet/parquet_leaf_reader_test.cpp +++ /dev/null @@ -1,535 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "format_v2/parquet/reader/parquet_leaf_reader.h" - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" - -namespace doris::format::parquet { -namespace { - -std::shared_ptr fixed_binary_array(const std::vector& values, - int byte_width) { - auto type = arrow::fixed_size_binary(byte_width); - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetLeafReader make_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type) { - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr); -} - -struct CapturedDecodedView { - DecodedValueKind value_kind = DecodedValueKind::INT32; - DecodedTimeUnit time_unit = DecodedTimeUnit::UNKNOWN; - int64_t row_count = 0; - int decimal_precision = -1; - int decimal_scale = -1; - int fixed_length = -1; - bool timestamp_is_adjusted_to_utc = false; - bool enable_strict_mode = false; - const cctz::time_zone* timezone = nullptr; - bool null_map_is_null = true; - std::vector null_map; - std::vector fixed_values; - std::vector binary_values; - std::vector owned_binary_values; -}; - -ParquetLeafReader make_spy_leaf_reader(ParquetTypeDescriptor descriptor, DataTypePtr type, - CapturedDecodedView* captured, - const cctz::time_zone* timezone = nullptr, - bool enable_strict_mode = false) { - auto appender = [captured](MutableColumnPtr&, const DecodedColumnView& view) { - captured->value_kind = view.value_kind; - captured->time_unit = view.time_unit; - captured->row_count = view.row_count; - captured->decimal_precision = view.decimal_precision; - captured->decimal_scale = view.decimal_scale; - captured->fixed_length = view.fixed_length; - captured->timestamp_is_adjusted_to_utc = view.timestamp_is_adjusted_to_utc; - captured->enable_strict_mode = view.enable_strict_mode; - captured->timezone = view.timezone; - captured->null_map_is_null = view.null_map == nullptr; - captured->null_map.clear(); - if (view.null_map != nullptr) { - captured->null_map.assign(view.null_map, view.null_map + view.row_count); - } - captured->fixed_values.clear(); - if (view.values != nullptr && view.value_kind == DecodedValueKind::INT64) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 8); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::FLOAT) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } else if (view.values != nullptr && view.value_kind == DecodedValueKind::INT32) { - captured->fixed_values.assign(view.values, view.values + view.row_count * 4); - } - captured->binary_values.clear(); - captured->owned_binary_values.clear(); - if (view.binary_values != nullptr) { - captured->owned_binary_values.reserve(view.binary_values->size()); - for (const auto& value : *view.binary_values) { - captured->owned_binary_values.emplace_back( - value.data == nullptr ? std::string() - : std::string(value.data, value.size)); - } - captured->binary_values.reserve(captured->owned_binary_values.size()); - for (const auto& value : captured->owned_binary_values) { - captured->binary_values.emplace_back(value.data(), value.size()); - } - } - return Status::OK(); - }; - return ParquetLeafReader(nullptr, descriptor, std::move(type), "leaf", nullptr, {}, timezone, - enable_strict_mode, std::move(appender)); -} - -} // namespace - -struct ParquetLeafReaderTestAccess { - static ParquetLeafBatch make_fixed_batch(const std::vector& def_levels, - const std::vector& rep_levels, - const std::vector& values, - bool read_dense_for_nullable = false) { - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._consumed_level_count = static_cast(def_levels.size()); - batch._decoded_level_count = static_cast(def_levels.size()); - batch._values_written = static_cast(values.size()); - batch._def_levels = def_levels.data(); - batch._rep_levels = rep_levels.data(); - batch._fixed_values = reinterpret_cast(values.data()); - batch._read_dense_for_nullable = read_dense_for_nullable; - return batch; - } - - static Status build_nested_batch(const ParquetLeafReader& reader, - const ParquetLeafBatch& leaf_batch, int64_t records_read, - int16_t value_slot_definition_level, - int16_t value_slot_repetition_level, - ParquetNestedScalarBatch* nested_batch) { - return reader.build_nested_batch_from_leaf_batch(leaf_batch, records_read, - value_slot_definition_level, nested_batch, - value_slot_repetition_level); - } - - static size_t binary_value_size(const ParquetLeafReader& reader) { - return reader._binary_values.size(); - } - - static size_t binary_value_capacity(const ParquetLeafReader& reader) { - return reader._binary_values.capacity(); - } -}; - -std::shared_ptr<::parquet::ColumnDescriptor> int32_column_descriptor(int16_t max_definition_level, - int16_t max_repetition_level) { - auto node = ::parquet::schema::PrimitiveNode::Make("leaf", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32); - return std::make_shared<::parquet::ColumnDescriptor>(node, max_definition_level, - max_repetition_level); -} - -ParquetLeafReader make_nested_leaf_reader( - const std::shared_ptr<::parquet::ColumnDescriptor>& descriptor, DataTypePtr type) { - ParquetTypeDescriptor type_descriptor; - type_descriptor.physical_type = ::parquet::Type::INT32; - type_descriptor.doris_type = type; - return ParquetLeafReader(descriptor.get(), type_descriptor, std::move(type), "nested_leaf", - nullptr); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesAreSpacedBeforeSerde) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30, 50}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& nullable = assert_cast(*column); - ASSERT_EQ(nullable.size(), 5); - EXPECT_FALSE(nullable.is_null_at(0)); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_FALSE(nullable.is_null_at(2)); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_FALSE(nullable.is_null_at(4)); - const auto& nested = assert_cast(nullable.get_nested_column()); - EXPECT_EQ(nested.get_element(0), 10); - EXPECT_EQ(nested.get_element(2), 30); - EXPECT_EQ(nested.get_element(4), 50); -} - -TEST(ParquetLeafReaderTest, DenseNullableFixedValuesRejectCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT32; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - const std::vector compact_values = {10, 30}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT32; - batch._fixed_values = reinterpret_cast(compact_values.data()); - batch._values_written = compact_values.size(); - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet values"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, Float16BinaryValuesAreConvertedToFloat) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - descriptor.fixed_length = 2; - auto type = std::make_shared(); - auto reader = make_leaf_reader(descriptor, type); - - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::FIXED_BINARY; - batch._binary_chunks = {fixed_binary_array( - {half(0x0000), half(0x8000), half(0x3E00), half(0x0001), half(0x7E00)}, 2)}; - batch._values_written = 5; - - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, nullptr, column); - ASSERT_TRUE(status.ok()) << status; - - const auto& floats = assert_cast(*column); - ASSERT_EQ(floats.size(), 5); - EXPECT_FLOAT_EQ(floats.get_element(0), 0.0F); - EXPECT_TRUE(std::signbit(floats.get_element(1))); - EXPECT_FLOAT_EQ(floats.get_element(2), 1.5F); - EXPECT_NEAR(floats.get_element(3), 5.9604645e-8F, 1e-12F); - EXPECT_TRUE(std::isnan(floats.get_element(4))); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableValuesAreSpacedWithNullRefs) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("aa").ok()); - ASSERT_TRUE(builder.Append("cc").ok()); - ASSERT_TRUE(builder.Append("ee").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 3; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 5, &null_map, column); - ASSERT_TRUE(status.ok()) << status; - - EXPECT_EQ(ParquetLeafReaderTestAccess::binary_value_size(reader), 0); - EXPECT_GE(ParquetLeafReaderTestAccess::binary_value_capacity(reader), 5); - - const auto& nullable = assert_cast(*column); - const auto& strings = assert_cast(nullable.get_nested_column()); - ASSERT_EQ(nullable.size(), 5); - EXPECT_EQ(strings.get_data_at(0).to_string(), "aa"); - EXPECT_TRUE(nullable.is_null_at(1)); - EXPECT_EQ(strings.get_data_at(2).to_string(), "cc"); - EXPECT_TRUE(nullable.is_null_at(3)); - EXPECT_EQ(strings.get_data_at(4).to_string(), "ee"); -} - -TEST(ParquetLeafReaderTest, ReleaseBinaryChunksDropsPayloadAndRetainsVectorCapacity) { - ParquetLeafBatch batch; - batch._binary_chunks.reserve(4); - auto array = fixed_binary_array({"payload"}, 7); - std::weak_ptr payload = array; - batch._binary_chunks.push_back(array); - array.reset(); - - const auto capacity = batch.binary_chunks().capacity(); - ASSERT_FALSE(payload.expired()); - batch.release_binary_chunks(); - - EXPECT_TRUE(payload.expired()); - EXPECT_TRUE(batch.binary_chunks().empty()); - EXPECT_EQ(batch.binary_chunks().capacity(), capacity); -} - -TEST(ParquetLeafReaderTest, BinaryDenseNullableRejectsCountMismatch) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - auto type = make_nullable(std::make_shared()); - auto reader = make_leaf_reader(descriptor, type); - - arrow::BinaryBuilder builder; - ASSERT_TRUE(builder.Append("only_one").ok()); - std::shared_ptr array; - ASSERT_TRUE(builder.Finish(&array).ok()); - - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::BINARY; - batch._binary_chunks = {array}; - batch._values_written = 1; - batch._read_dense_for_nullable = true; - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - auto status = reader.append_values(batch, 3, &null_map, column); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid dense nullable parquet binary values"), - std::string::npos); - EXPECT_EQ(ParquetLeafReaderTestAccess::binary_value_size(reader), 0); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCarriesDescriptorSessionAndNullMapFields) { - ParquetTypeDescriptor descriptor; - descriptor.physical_type = ::parquet::Type::INT64; - descriptor.time_unit = ParquetTimeUnit::NANOS; - descriptor.decimal_precision = 18; - descriptor.decimal_scale = 4; - descriptor.fixed_length = 12; - descriptor.timestamp_is_adjusted_to_utc = true; - auto type = make_nullable(std::make_shared()); - cctz::time_zone shanghai; - ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); - - CapturedDecodedView captured; - auto reader = make_spy_leaf_reader(descriptor, type, &captured, &shanghai, true); - const std::vector values = {100, 200, 300}; - ParquetLeafBatch batch; - batch._value_kind = DecodedValueKind::INT64; - batch._fixed_values = reinterpret_cast(values.data()); - batch._values_written = values.size(); - - const NullMap null_map = {0, 1, 0}; - auto column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, &null_map, column).ok()); - EXPECT_EQ(captured.value_kind, DecodedValueKind::INT64); - EXPECT_EQ(captured.time_unit, DecodedTimeUnit::NANOS); - EXPECT_EQ(captured.row_count, 3); - EXPECT_EQ(captured.decimal_precision, 18); - EXPECT_EQ(captured.decimal_scale, 4); - EXPECT_EQ(captured.fixed_length, 12); - EXPECT_TRUE(captured.timestamp_is_adjusted_to_utc); - EXPECT_TRUE(captured.enable_strict_mode); - EXPECT_EQ(captured.timezone, &shanghai); - EXPECT_FALSE(captured.null_map_is_null); - EXPECT_EQ(captured.null_map, std::vector({0, 1, 0})); - - auto required_column = type->create_column(); - ASSERT_TRUE(reader.append_values(batch, 3, nullptr, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); - - const NullMap empty_null_map; - ASSERT_TRUE(reader.append_values(batch, 3, &empty_null_map, required_column).ok()); - EXPECT_TRUE(captured.null_map_is_null); -} - -TEST(ParquetLeafReaderTest, DecodedColumnViewCapturesBinaryFixedLengthAndFloat16Override) { - ParquetTypeDescriptor binary_descriptor; - binary_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - binary_descriptor.fixed_length = 4; - auto type = std::make_shared(); - - CapturedDecodedView binary_view; - auto binary_reader = make_spy_leaf_reader(binary_descriptor, type, &binary_view); - ParquetLeafBatch binary_batch; - binary_batch._value_kind = DecodedValueKind::FIXED_BINARY; - binary_batch._binary_chunks = {fixed_binary_array({"abcd", "wxyz"}, 4)}; - binary_batch._values_written = 2; - auto binary_column = type->create_column(); - ASSERT_TRUE(binary_reader.append_values(binary_batch, 2, nullptr, binary_column).ok()); - EXPECT_EQ(binary_view.value_kind, DecodedValueKind::FIXED_BINARY); - EXPECT_EQ(binary_view.fixed_length, 4); - ASSERT_EQ(binary_view.owned_binary_values.size(), 2); - EXPECT_EQ(binary_view.owned_binary_values[0], "abcd"); - EXPECT_EQ(binary_view.owned_binary_values[1], "wxyz"); - - ParquetTypeDescriptor float16_descriptor; - float16_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - float16_descriptor.extra_type_info = ParquetExtraTypeInfo::FLOAT16; - float16_descriptor.fixed_length = 2; - CapturedDecodedView float16_view; - auto float16_reader = make_spy_leaf_reader(float16_descriptor, - std::make_shared(), &float16_view); - auto half = [](uint16_t value) { - std::string bytes(sizeof(value), '\0'); - memcpy(bytes.data(), &value, sizeof(value)); - return bytes; - }; - ParquetLeafBatch float16_batch; - float16_batch._value_kind = DecodedValueKind::FIXED_BINARY; - float16_batch._binary_chunks = {fixed_binary_array({half(0x3E00), half(0x4000)}, 2)}; - float16_batch._values_written = 2; - auto float16_column = std::make_shared()->create_column(); - ASSERT_TRUE(float16_reader.append_values(float16_batch, 2, nullptr, float16_column).ok()); - EXPECT_EQ(float16_view.value_kind, DecodedValueKind::FLOAT); - ASSERT_EQ(float16_view.fixed_values.size(), sizeof(float) * 2); - const auto* floats = reinterpret_cast(float16_view.fixed_values.data()); - EXPECT_FLOAT_EQ(floats[0], 1.5F); - EXPECT_FLOAT_EQ(floats[1], 2.0F); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLevels) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 2, 2}; - const std::vector rep_levels = {0, 1, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 2, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.records_read, 2); - EXPECT_EQ(nested_batch.levels_written, 3); - EXPECT_EQ(nested_batch.value_indices, std::vector({0, 1, 2})); - const auto& nested_values = assert_cast(*nested_batch.values_column); - ASSERT_EQ(nested_values.size(), 3); - EXPECT_EQ(nested_values.get_element(0), 10); - EXPECT_EQ(nested_values.get_element(2), 30); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutValueSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 777, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 2, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutLeafValues) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 1, 2, 0}; - const std::vector rep_levels = {0, 1, 0, 0}; - const std::vector values = {10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 3, 1, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({0, -1, 1, -1})); -} - -TEST(ParquetLeafReaderTest, NestedBatchValueLayoutPayloadSlots) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {1, 2, 0, 2}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {777, 10, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(nested_batch.value_indices, std::vector({-1, 1, -1, 2})); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsMismatchedValueLayout) { - auto descriptor = int32_column_descriptor(2, 1); - auto reader = make_nested_leaf_reader(descriptor, std::make_shared()); - const std::vector def_levels = {2, 0, 2, 0}; - const std::vector rep_levels = {0, 0, 0, 0}; - const std::vector values = {10, 20, 30}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 4, 2, 1, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("inconsistent value count"), std::string::npos); -} - -TEST(ParquetLeafReaderTest, NestedBatchRejectsDenseNullable) { - auto descriptor = int32_column_descriptor(1, 0); - auto reader = - make_nested_leaf_reader(descriptor, make_nullable(std::make_shared())); - const std::vector def_levels = {1}; - const std::vector rep_levels = {0}; - const std::vector values = {10}; - const auto leaf_batch = - ParquetLeafReaderTestAccess::make_fixed_batch(def_levels, rep_levels, values, true); - - ParquetNestedScalarBatch nested_batch; - const auto status = ParquetLeafReaderTestAccess::build_nested_batch(reader, leaf_batch, 1, 0, 0, - &nested_batch); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Dense nullable parquet nested reader is not supported"), - std::string::npos); -} - -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index b7a2b34fdbe024..f5c0dc7c3b9ce0 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -5,9 +5,7 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at -// // http://www.apache.org/licenses/LICENSE-2.0 -// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,188 +13,47 @@ // specific language governing permissions and limitations // under the License. -#include #include -#include #include #include #include #include -#include #include #include "common/consts.h" #include "core/assert_cast.h" -#include "core/column/column_array.h" -#include "core/column/column_map.h" -#include "core/column/column_nullable.h" #include "core/column/column_string.h" -#include "core/column/column_struct.h" #include "core/column/column_vector.h" -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" -#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_struct.h" -#include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" -#include "format_v2/parquet/reader/list_column_reader.h" -#include "format_v2/parquet/reader/map_column_reader.h" -#include "format_v2/parquet/reader/nested_column_materializer.h" #include "format_v2/parquet/reader/row_position_column_reader.h" -#include "format_v2/parquet/reader/scalar_column_reader.h" -#include "format_v2/parquet/reader/struct_column_reader.h" #include "format_v2/parquet/selection_vector.h" #include "storage/utils.h" namespace doris::format::parquet { namespace { -ParquetColumnSchema int64_schema(std::string name = "mock") { +ParquetColumnSchema int64_schema() { ParquetColumnSchema schema; schema.local_id = 0; - schema.name = std::move(name); + schema.name = "mock"; schema.type = std::make_shared(); return schema; } -ParquetColumnSchema nested_int64_schema(std::string name, int16_t nullable_definition_level, - int16_t definition_level, int16_t repetition_level = 0, - int16_t repeated_ancestor_definition_level = 0) { - ParquetColumnSchema schema = int64_schema(std::move(name)); - schema.type = make_nullable(std::make_shared()); - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - return schema; -} - -ParquetColumnSchema nested_struct_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "struct"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})); - return schema; -} - -ParquetColumnSchema nested_list_schema(std::string name, DataTypePtr element_type, - int16_t nullable_definition_level, int16_t definition_level, - int16_t repetition_level, - int16_t repeated_ancestor_definition_level) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.kind = ParquetColumnSchemaKind::LIST; - schema.nullable_definition_level = nullable_definition_level; - schema.definition_level = definition_level; - schema.repetition_level = repetition_level; - schema.repeated_repetition_level = repetition_level; - schema.repeated_ancestor_definition_level = repeated_ancestor_definition_level; - schema.type = make_nullable(std::make_shared(std::move(element_type))); - return schema; -} - -ParquetColumnSchema nested_map_schema( - DataTypePtr value_type = make_nullable(std::make_shared())) { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "map"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.nullable_definition_level = 1; - schema.definition_level = 2; - schema.repetition_level = 1; - schema.repeated_ancestor_definition_level = 2; - schema.type = make_nullable(std::make_shared( - make_nullable(std::make_shared()), std::move(value_type))); - return schema; -} - -ParquetColumnSchema bare_repeated_int64_list_schema() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "repeated"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.definition_level = 1; - schema.repetition_level = 1; - schema.repeated_repetition_level = 1; - schema.repeated_ancestor_definition_level = 1; - schema.type = std::make_shared(std::make_shared()); - return schema; -} - -std::unique_ptr primitive_child(int local_id, std::string name, - DataTypePtr type) { - auto child = std::make_unique(); - child->local_id = local_id; - child->name = std::move(name); - child->kind = ParquetColumnSchemaKind::PRIMITIVE; - child->leaf_column_id = local_id; - child->type = std::move(type); - child->type_descriptor.physical_type = ::parquet::Type::INT32; - child->type_descriptor.doris_type = child->type; - return child; -} - -ParquetColumnSchema struct_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "s"; - schema.kind = ParquetColumnSchemaKind::STRUCT; - schema.children.push_back(primitive_child(0, "a", std::make_shared())); - schema.children.push_back(primitive_child(1, "b", std::make_shared())); - DataTypes types = {make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)}; - Strings names = {"a", "b"}; - schema.type = std::make_shared(types, names); - return schema; -} - -ParquetColumnSchema list_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "xs"; - schema.kind = ParquetColumnSchemaKind::LIST; - schema.children.push_back(primitive_child(0, "element", std::make_shared())); - schema.type = std::make_shared(schema.children[0]->type); - return schema; -} - -ParquetColumnSchema map_schema_for_projection() { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = "m"; - schema.kind = ParquetColumnSchemaKind::MAP; - schema.children.push_back(primitive_child(0, "key", std::make_shared())); - schema.children.push_back(primitive_child(1, "value", std::make_shared())); - schema.type = std::make_shared(make_nullable(schema.children[0]->type), - make_nullable(schema.children[1]->type)); - return schema; -} - class CursorColumnReader final : public ParquetColumnReader { public: CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - if (column.get() == nullptr || rows_read == nullptr) { - return Status::InvalidArgument("invalid mock read arguments"); - } - auto* values = assert_cast(column.get()); + DORIS_CHECK(column != nullptr); + DORIS_CHECK(rows_read != nullptr); + auto& values = assert_cast(*column); for (int64_t row = 0; row < rows; ++row) { - values->insert_value(_cursor + row); + values.insert_value(_cursor + row); } _read_lengths.push_back(rows); _cursor += rows; @@ -205,6 +62,7 @@ class CursorColumnReader final : public ParquetColumnReader { } Status skip(int64_t rows) override { + DORIS_CHECK(rows >= 0); _skip_lengths.push_back(rows); _cursor += rows; return Status::OK(); @@ -220,269 +78,6 @@ class CursorColumnReader final : public ParquetColumnReader { std::vector _read_lengths; }; -class ScriptedNestedReader final : public ParquetColumnReader { -public: - ScriptedNestedReader(ParquetColumnSchema schema, DataTypePtr type, - std::vector def_levels, std::vector rep_levels, - bool has_repeated_child = false, bool build_nulls = false) - : ParquetColumnReader(schema, std::move(type)), - _def_levels(std::move(def_levels)), - _rep_levels(std::move(rep_levels)), - _has_repeated_child(has_repeated_child), - _build_nulls(build_nulls) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - _build_lengths.push_back(length_upper_bound); - if (column.get() == nullptr || values_read == nullptr) { - return Status::InvalidArgument("invalid scripted nested build arguments"); - } - for (int64_t row = 0; row < length_upper_bound; ++row) { - insert_value(column, _next_value++, _build_nulls); - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - set_nested_build_level_cursor(std::min(nested_build_level_cursor() + length_upper_bound, - static_cast(_def_levels.size()))); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return _has_repeated_child; } - - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - -private: - static void insert_value(MutableColumnPtr& column, int64_t value, bool is_null) { - if (auto* nullable_column = check_and_get_column(*column); - nullable_column != nullptr) { - if (is_null) { - nullable_column->insert_default(); - return; - } - assert_cast(nullable_column->get_nested_column()).insert_value(value); - nullable_column->get_null_map_data().push_back(0); - return; - } - assert_cast(*column).insert_value(value); - } - - std::vector _def_levels; - std::vector _rep_levels; - bool _has_repeated_child = false; - bool _build_nulls = false; - int64_t _next_value = 0; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; -}; - -class ChunkedNestedLeafReader final : public ParquetColumnReader { -public: - ChunkedNestedLeafReader() - : ParquetColumnReader(nested_int64_schema("element", 0, 1, 1, 1), - std::make_shared()) {} - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } - - Status load_nested_batch(int64_t rows) override { - _load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status load_nested_levels_batch(int64_t rows) override { - _level_load_lengths.push_back(rows); - _def_levels.assign(static_cast(rows), 1); - _rep_levels.assign(static_cast(rows), 0); - return Status::OK(); - } - - Status build_nested_column(int64_t length_upper_bound, MutableColumnPtr& column, - int64_t* values_read) override { - DORIS_CHECK(column.get() != nullptr); - DORIS_CHECK(values_read != nullptr); - _initial_column_sizes.push_back(column->size()); - _build_lengths.push_back(length_upper_bound); - if (auto* nullable = check_and_get_column(*column); nullable != nullptr) { - auto& values = assert_cast(nullable->get_nested_column()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values.insert_value(row); - nullable->get_null_map_data().push_back(0); - } - } else { - auto* values = assert_cast(column.get()); - for (int64_t row = 0; row < length_upper_bound; ++row) { - values->insert_value(row); - } - } - *values_read = length_upper_bound; - return Status::OK(); - } - - Status consume_nested_column(int64_t length_upper_bound, int64_t* values_consumed) override { - DORIS_CHECK(values_consumed != nullptr); - _consume_lengths.push_back(length_upper_bound); - *values_consumed = length_upper_bound; - return Status::OK(); - } - - const std::vector& nested_definition_levels() const override { return _def_levels; } - const std::vector& nested_repetition_levels() const override { return _rep_levels; } - int64_t nested_levels_written() const override { - return static_cast(_def_levels.size()); - } - bool is_or_has_repeated_child() const override { return true; } - - const std::vector& load_lengths() const { return _load_lengths; } - const std::vector& build_lengths() const { return _build_lengths; } - const std::vector& consume_lengths() const { return _consume_lengths; } - const std::vector& level_load_lengths() const { return _level_load_lengths; } - const std::vector& initial_column_sizes() const { return _initial_column_sizes; } - -private: - std::vector _def_levels; - std::vector _rep_levels; - std::vector _load_lengths; - std::vector _level_load_lengths; - std::vector _build_lengths; - std::vector _consume_lengths; - std::vector _initial_column_sizes; -}; - -} // namespace - -struct ScalarColumnReaderTestAccess { - static void set_nested_batch(ScalarColumnReader* reader, - std::unique_ptr batch) { - reader->_nested_batch = std::move(batch); - } - - static int64_t page_filtered_rows_to_skip(const ScalarColumnReader& reader, int64_t rows) { - return reader.page_filtered_rows_to_skip(rows); - } - - static void set_row_group_rows_read(ScalarColumnReader* reader, int64_t rows) { - reader->_row_group_rows_read = rows; - } - - static const ParquetLeafReader* leaf_reader_address(ScalarColumnReader& reader) { - return &reader.leaf_reader(); - } - - static size_t dictionary_binary_capacity(const ScalarColumnReader& reader) { - return reader._dictionary_binary_values.capacity(); - } - - static size_t dictionary_binary_size(const ScalarColumnReader& reader) { - return reader._dictionary_binary_values.size(); - } - - static Status append_dictionary_filtered_values( - ScalarColumnReader& reader, const std::vector>& chunks, - const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, - IColumn::Filter* row_filter, int64_t* matched_rows, bool* used_filter) { - return reader.append_dictionary_filtered_values(chunks, dictionary_filter, column, - row_filter, matched_rows, used_filter); - } -}; - -namespace { - -ParquetColumnSchema string_schema(std::string name = "string") { - ParquetColumnSchema schema; - schema.local_id = 0; - schema.name = std::move(name); - schema.type = std::make_shared(); - schema.type_descriptor.physical_type = ::parquet::Type::BYTE_ARRAY; - schema.type_descriptor.doris_type = schema.type; - return schema; -} - -std::shared_ptr<::arrow::Array> dictionary_array(const std::vector& indices, - const std::vector& values) { - ::arrow::Int8Builder index_builder; - EXPECT_TRUE(index_builder.AppendValues(indices).ok()); - auto index_result = index_builder.Finish(); - EXPECT_TRUE(index_result.ok()) << index_result.status(); - - ::arrow::StringBuilder dictionary_builder; - EXPECT_TRUE(dictionary_builder.AppendValues(values).ok()); - auto dictionary_result = dictionary_builder.Finish(); - EXPECT_TRUE(dictionary_result.ok()) << dictionary_result.status(); - - auto result = ::arrow::DictionaryArray::FromArrays( - ::arrow::dictionary(::arrow::int8(), ::arrow::utf8()), *index_result, - *dictionary_result); - EXPECT_TRUE(result.ok()) << result.status(); - return *result; -} - -std::unique_ptr make_scripted_scalar_reader( - ParquetColumnSchema schema, std::unique_ptr batch) { - auto reader = std::make_unique(schema, nullptr); - ScalarColumnReaderTestAccess::set_nested_batch(reader.get(), std::move(batch)); - return reader; -} - -std::unique_ptr scalar_batch(std::vector def_levels, - std::vector rep_levels, - std::vector value_indices, - std::vector values) { - auto batch = std::make_unique(); - batch->levels_written = static_cast(def_levels.size()); - batch->def_levels = std::move(def_levels); - batch->rep_levels = std::move(rep_levels); - batch->value_indices = std::move(value_indices); - auto column = ColumnInt64::create(); - for (const auto value : values) { - column->insert_value(value); - } - batch->values_column = std::move(column); - return batch; -} - -class DefaultOnlyReader final : public ParquetColumnReader { -public: - DefaultOnlyReader() - : ParquetColumnReader(int64_schema("default_only"), std::make_shared()) { - } - - Status read(int64_t, MutableColumnPtr&, int64_t*) override { - return Status::NotSupported("unused"); - } -}; - GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { const auto ref = column.get_data_at(row); EXPECT_EQ(ref.size, sizeof(GlobalRowLoacationV2)); @@ -491,50 +86,6 @@ GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { return location; } -TEST(ParquetScalarColumnReaderTest, DictionaryIndexOutsideFilterIsCorruption) { - ScalarColumnReader reader(string_schema("dictionary_value"), nullptr); - MutableColumnPtr column = ColumnString::create(); - IColumn::Filter row_filter; - int64_t matched_rows = 0; - bool used_filter = false; - const std::vector> chunks = { - dictionary_array({0, 1}, {"keep", "out-of-range"})}; - - const auto status = ScalarColumnReaderTestAccess::append_dictionary_filtered_values( - reader, chunks, IColumn::Filter {1}, column, &row_filter, &matched_rows, &used_filter); - EXPECT_EQ(ErrorCode::CORRUPTION, status.code()) << status; - EXPECT_NE(status.to_string().find("Invalid parquet dictionary index 1"), std::string::npos); - EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_size(reader), 0); -} - -TEST(ParquetScalarColumnReaderTest, LeafReaderAndDictionaryScratchArePersistent) { - ScalarColumnReader reader(string_schema("persistent_leaf"), nullptr); - const auto* leaf_reader = ScalarColumnReaderTestAccess::leaf_reader_address(reader); - - MutableColumnPtr column = ColumnString::create(); - IColumn::Filter row_filter; - int64_t matched_rows = 0; - bool used_filter = false; - const std::vector> chunks = { - dictionary_array({0, 1}, {"first", "second"})}; - ASSERT_TRUE(ScalarColumnReaderTestAccess::append_dictionary_filtered_values( - reader, chunks, IColumn::Filter {1, 1}, column, &row_filter, &matched_rows, - &used_filter) - .ok()); - const auto retained_capacity = ScalarColumnReaderTestAccess::dictionary_binary_capacity(reader); - EXPECT_GE(retained_capacity, 2); - EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_size(reader), 0); - - row_filter.clear(); - ASSERT_TRUE(ScalarColumnReaderTestAccess::append_dictionary_filtered_values( - reader, chunks, IColumn::Filter {1, 0}, column, &row_filter, &matched_rows, - &used_filter) - .ok()); - EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_capacity(reader), retained_capacity); - EXPECT_EQ(ScalarColumnReaderTestAccess::dictionary_binary_size(reader), 0); - EXPECT_EQ(ScalarColumnReaderTestAccess::leaf_reader_address(reader), leaf_reader); -} - } // namespace TEST(SelectionVectorTest, IdentitySelectionToRanges) { @@ -575,30 +126,6 @@ TEST(SelectionVectorTest, OutputRangesReuseCapacity) { EXPECT_EQ(ranges.capacity(), retained_capacity); } -TEST(ParquetNestedScalarBatchTest, ResetRetainsScratchCapacityAndColumn) { - ParquetNestedScalarBatch batch; - batch.def_levels.reserve(16); - batch.rep_levels.reserve(16); - batch.value_indices.reserve(16); - batch.def_levels.push_back(1); - batch.rep_levels.push_back(0); - batch.value_indices.push_back(0); - batch.values_column = ColumnInt64::create(); - batch.values_column->insert_default(); - const auto* values_column = batch.values_column.get(); - - batch.reset(); - - EXPECT_TRUE(batch.def_levels.empty()); - EXPECT_TRUE(batch.rep_levels.empty()); - EXPECT_TRUE(batch.value_indices.empty()); - EXPECT_GE(batch.def_levels.capacity(), 16); - EXPECT_GE(batch.rep_levels.capacity(), 16); - EXPECT_GE(batch.value_indices.capacity(), 16); - EXPECT_EQ(batch.values_column.get(), values_column); - EXPECT_TRUE(batch.values_column->empty()); -} - TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { SelectionVector selection(2); EXPECT_FALSE(selection.verify(3, 3).ok()); @@ -638,573 +165,12 @@ TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { SelectionVector selection; auto column = std::make_shared()->create_column(); ASSERT_TRUE(reader.select(selection, 0, 4, column).ok()); - EXPECT_EQ(column->size(), 0); + EXPECT_TRUE(column->empty()); EXPECT_EQ(reader.cursor(), 4); EXPECT_TRUE(reader.read_lengths().empty()); EXPECT_EQ(reader.skip_lengths(), std::vector({4})); } -TEST(ParquetColumnReaderControlTest, BaseNestedDefaultsAndSkipNested) { - DefaultOnlyReader base_reader; - EXPECT_FALSE(base_reader.skip(1).ok()); - EXPECT_FALSE(base_reader.load_nested_batch(1).ok()); - - auto column = std::make_shared()->create_column(); - int64_t values_read = 0; - EXPECT_FALSE(base_reader.build_nested_column(1, column, &values_read).ok()); - - int64_t values_consumed = 0; - EXPECT_FALSE(base_reader.consume_nested_column(1, &values_consumed).ok()); -} - -TEST(ParquetColumnReaderControlTest, NestedSkipConsumesBoundedBatchesWithoutMaterializing) { - auto element_reader = std::make_unique(); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - ASSERT_TRUE(reader.skip(8193).ok()); - EXPECT_TRUE(element_reader_ptr->load_lengths().empty()); - EXPECT_EQ(element_reader_ptr->level_load_lengths(), std::vector({4096, 4096, 1})); - EXPECT_EQ(element_reader_ptr->consume_lengths(), std::vector({4096, 4096, 1})); - EXPECT_TRUE(element_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(element_reader_ptr->initial_column_sizes().empty()); -} - -TEST(ParquetColumnReaderControlTest, MapSkipConsumesBothStreamsWithoutMaterializing) { - const std::vector def_levels {3, 3, 1, 3}; - const std::vector rep_levels {0, 1, 0, 0}; - auto key_reader = std::make_unique( - nested_int64_schema("key", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* key_reader_ptr = key_reader.get(); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), def_levels, rep_levels); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(key_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(key_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_EQ(value_reader_ptr->consume_lengths(), std::vector({3})); - EXPECT_TRUE(key_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(value_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, StructSkipConsumesNullSeparatedChildSpans) { - const std::vector def_levels {2, 0, 2}; - const std::vector rep_levels {0, 0, 0}; - auto shape_reader = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* shape_reader_ptr = shape_reader.get(); - auto child_reader = std::make_unique( - nested_int64_schema("child", 1, 2), make_nullable(std::make_shared()), - def_levels, rep_levels); - auto* child_reader_ptr = child_reader.get(); - std::vector> children; - children.push_back(std::move(shape_reader)); - children.push_back(std::move(child_reader)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {-1, 0}); - - ASSERT_TRUE(reader.skip(3).ok()); - EXPECT_EQ(shape_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(child_reader_ptr->level_load_lengths(), std::vector({3})); - EXPECT_EQ(shape_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_EQ(child_reader_ptr->consume_lengths(), std::vector({1, 1})); - EXPECT_TRUE(shape_reader_ptr->build_lengths().empty()); - EXPECT_TRUE(child_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipConsumesRecursivelyWithoutMaterializing) { - auto leaf_reader = std::make_unique( - nested_int64_schema("leaf", 0, 1, 1, 1), std::make_shared(), - std::vector {1, 1}, std::vector {0, 0}); - auto* leaf_reader_ptr = leaf_reader.get(); - const auto inner_type = std::make_shared(std::make_shared()); - auto inner_reader = std::make_unique(bare_repeated_int64_list_schema(), - inner_type, std::move(leaf_reader)); - const auto outer_type = std::make_shared(inner_type); - ListColumnReader reader(bare_repeated_int64_list_schema(), outer_type, std::move(inner_reader)); - - ASSERT_TRUE(reader.skip(2).ok()); - EXPECT_EQ(leaf_reader_ptr->level_load_lengths(), std::vector({2})); - EXPECT_EQ(leaf_reader_ptr->consume_lengths(), std::vector({2})); - EXPECT_TRUE(leaf_reader_ptr->build_lengths().empty()); -} - -TEST(ParquetColumnReaderControlTest, NestedMaterializerHelpersAppendOffsetsAndParentNulls) { - ColumnArray::Offsets64 offsets; - append_offsets(offsets, {3, 0, 2}); - ASSERT_EQ(offsets.size(), 3); - EXPECT_EQ(offsets[0], 3); - EXPECT_EQ(offsets[1], 3); - EXPECT_EQ(offsets[2], 5); - append_offsets(offsets, {1, 4}); - ASSERT_EQ(offsets.size(), 5); - EXPECT_EQ(offsets[3], 6); - EXPECT_EQ(offsets[4], 10); - - const NullMap parent_nulls = {0, 1, 0}; - append_parent_nulls(nullptr, parent_nulls); - NullMap dst = {1}; - append_parent_nulls(&dst, parent_nulls); - EXPECT_EQ(dst, NullMap({1, 0, 1, 0})); -} - -TEST(ParquetColumnReaderControlTest, PageFilteredRowsToSkipUsesOnlyFullSkippedRanges) { - ParquetPageSkipPlan page_skip_plan; - page_skip_plan.skipped_ranges = {RowRange {0, 3}, RowRange {5, 2}, RowRange {10, 4}}; - - auto schema = nested_int64_schema("page_filtered", 0, 0); - ScalarColumnReader reader(schema, nullptr, &page_skip_plan); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 3), 3); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 3); - - ScalarColumnReaderTestAccess::set_row_group_rows_read(&reader, 5); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 2), 2); - EXPECT_EQ(ScalarColumnReaderTestAccess::page_filtered_rows_to_skip(reader, 5), 2); -} - -TEST(ParquetColumnReaderControlTest, StructSkipsNullParentForRepeatedChildAndBatchesPresentRows) { - auto repeated_child = std::make_unique( - nested_int64_schema("repeated_shape", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2, 2, 2}, - std::vector {0, 0, 0, 0}, true); - auto* repeated_child_ptr = repeated_child.get(); - auto scalar_child = make_scripted_scalar_reader( - nested_int64_schema("scalar_child", 1, 2), - scalar_batch({2, 0, 2, 2}, {0, 0, 0, 0}, {0, -1, 1, 2}, {10, 20, 30})); - auto* scalar_child_ptr = scalar_child.get(); - - std::vector> children; - children.push_back(std::move(repeated_child)); - children.push_back(std::move(scalar_child)); - StructColumnReader reader(nested_struct_schema(), - make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared()), - make_nullable(std::make_shared())}, - Strings {"a", "b"})), - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_FALSE(nullable_column.is_null_at(3)); - EXPECT_EQ(repeated_child_ptr->build_lengths(), std::vector({1, 2})); - EXPECT_EQ(scalar_child_ptr->nested_build_level_cursor(), 4); -} - -TEST(ParquetColumnReaderControlTest, StructFallsBackToFirstChildWhenAllChildrenAreRepeated) { - auto first_child = std::make_unique( - nested_int64_schema("first", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 0}, std::vector {0, 0}, true); - auto second_child = std::make_unique( - nested_int64_schema("second", 1, 2, 1), - make_nullable(std::make_shared()), std::vector {2, 2}, - std::vector {0, 0}, true); - - std::vector> children; - children.push_back(std::move(first_child)); - children.push_back(std::move(second_child)); - StructColumnReader reader(nested_struct_schema(), nested_struct_schema().type, - std::move(children), {0, 1}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(rows_read, 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesComplexChildShapeOnly) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 2, 0, 0, 2, 2}, std::vector {0, 0, 0, 0, 0, 0}); - - ParquetColumnSchema map_schema = nested_map_schema(); - map_schema.nullable_definition_level = 2; - map_schema.definition_level = 3; - map_schema.repeated_ancestor_definition_level = 0; - auto key_reader = std::make_unique( - nested_int64_schema("key", 3, 3, 1, 0), - make_nullable(std::make_shared()), - std::vector {3, 3, 0, 0, 3, 3}, std::vector {0, 0, 0, 0, 0, 0}); - auto value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 4, 4, 1, 0), - scalar_batch({4, 4, 0, 0, 4, 4}, {0, 0, 0, 0, 0, 0}, - {0, 1, -1, -1, 2, 3}, {10, 20, 30, 40})); - auto map_reader = std::make_unique( - map_schema, map_schema.type, std::move(key_reader), std::move(value_reader)); - - std::vector> children; - children.push_back(std::move(shape_child)); - children.push_back(std::move(map_reader)); - auto struct_type = make_nullable(std::make_shared(DataTypes {map_schema.type}, - Strings {"partitionValues"})); - StructColumnReader reader(nested_struct_schema(), struct_type, std::move(children), {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(6, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 6); - - const auto& nullable_struct = assert_cast(*column); - ASSERT_EQ(nullable_struct.size(), 6); - EXPECT_FALSE(nullable_struct.is_null_at(0)); - EXPECT_FALSE(nullable_struct.is_null_at(1)); - EXPECT_TRUE(nullable_struct.is_null_at(2)); - EXPECT_TRUE(nullable_struct.is_null_at(3)); - EXPECT_FALSE(nullable_struct.is_null_at(4)); - EXPECT_FALSE(nullable_struct.is_null_at(5)); - - const auto& struct_column = - assert_cast(nullable_struct.get_nested_column()); - const auto& map_nullable = assert_cast(struct_column.get_column(0)); - ASSERT_EQ(map_nullable.size(), 6); - EXPECT_FALSE(map_nullable.is_null_at(0)); - EXPECT_FALSE(map_nullable.is_null_at(1)); - EXPECT_TRUE(map_nullable.is_null_at(2)); - EXPECT_TRUE(map_nullable.is_null_at(3)); - EXPECT_FALSE(map_nullable.is_null_at(4)); - EXPECT_FALSE(map_nullable.is_null_at(5)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 6); - EXPECT_EQ(map_column.get_offsets()[0], 1); - EXPECT_EQ(map_column.get_offsets()[1], 2); - EXPECT_EQ(map_column.get_offsets()[2], 2); - EXPECT_EQ(map_column.get_offsets()[3], 2); - EXPECT_EQ(map_column.get_offsets()[4], 3); - EXPECT_EQ(map_column.get_offsets()[5], 4); -} - -TEST(ParquetColumnReaderControlTest, StructNullParentAdvancesNestedStructDescendants) { - auto shape_child = std::make_unique( - nested_int64_schema("shape", 1, 2), make_nullable(std::make_shared()), - std::vector {2, 0, 2}, std::vector {0, 0, 0}); - - auto id_batch = scalar_batch({4, 3, 4}, {0, 0, 0}, {0, -1, 1}, {10, 20}); - id_batch->value_slot_definition_level = 3; - auto id_reader = - make_scripted_scalar_reader(nested_int64_schema("id", 3, 4), std::move(id_batch)); - - ParquetColumnSchema inner_schema; - inner_schema.local_id = 0; - inner_schema.name = "stats_parsed"; - inner_schema.kind = ParquetColumnSchemaKind::STRUCT; - inner_schema.nullable_definition_level = 2; - inner_schema.definition_level = 3; - inner_schema.type = make_nullable(std::make_shared( - DataTypes {make_nullable(std::make_shared())}, Strings {"id"})); - - std::vector> inner_children; - inner_children.push_back(std::move(id_reader)); - auto inner_reader = std::make_unique( - inner_schema, inner_schema.type, std::move(inner_children), std::vector {0}); - - std::vector> outer_children; - outer_children.push_back(std::move(shape_child)); - outer_children.push_back(std::move(inner_reader)); - auto outer_type = make_nullable(std::make_shared(DataTypes {inner_schema.type}, - Strings {"stats_parsed"})); - StructColumnReader reader(nested_struct_schema(), outer_type, std::move(outer_children), - {-1, 0}); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& outer_nullable = assert_cast(*column); - ASSERT_EQ(outer_nullable.size(), 3); - EXPECT_FALSE(outer_nullable.is_null_at(0)); - EXPECT_TRUE(outer_nullable.is_null_at(1)); - EXPECT_FALSE(outer_nullable.is_null_at(2)); - - const auto& outer_struct = assert_cast(outer_nullable.get_nested_column()); - const auto& inner_nullable = assert_cast(outer_struct.get_column(0)); - ASSERT_EQ(inner_nullable.size(), 3); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_TRUE(inner_nullable.is_null_at(1)); - EXPECT_FALSE(inner_nullable.is_null_at(2)); - - const auto& inner_struct = assert_cast(inner_nullable.get_nested_column()); - const auto& id_nullable = assert_cast(inner_struct.get_column(0)); - const auto& id_values = assert_cast(id_nullable.get_nested_column()); - EXPECT_EQ(id_values.get_element(0), 10); - EXPECT_EQ(id_values.get_element(2), 20); -} - -TEST(ParquetColumnReaderControlTest, ListKeepsEmptyBareRepeatedPrimitiveRows) { - auto element_reader = std::make_unique( - nested_int64_schema("element", 0, 1, 1, 1), std::make_shared(), - std::vector {0, 1, 1, 0}, std::vector {0, 0, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - ListColumnReader reader(bare_repeated_int64_list_schema(), - bare_repeated_int64_list_schema().type, std::move(element_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(3, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 3); - - const auto& array_column = assert_cast(*column); - ASSERT_EQ(array_column.get_offsets().size(), 3); - EXPECT_EQ(array_column.get_offsets()[0], 0); - EXPECT_EQ(array_column.get_offsets()[1], 2); - EXPECT_EQ(array_column.get_offsets()[2], 2); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({2})); -} - -TEST(ParquetColumnReaderControlTest, NestedListSkipsAncestorEmptyRowsButKeepsNullElements) { - auto element_reader = - std::make_unique(nested_int64_schema("element", 5, 5, 2, 4), - make_nullable(std::make_shared()), - std::vector {1, 5, 5, 5, 2, 5, 2, 0}, - std::vector {0, 0, 2, 1, 0, 1, 1, 0}); - auto* element_reader_ptr = element_reader.get(); - - const auto inner_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()))); - auto inner_reader = std::make_unique( - nested_list_schema("inner", make_nullable(std::make_shared()), 3, 4, 2, - 2), - inner_type, std::move(element_reader)); - auto outer_type = make_nullable(std::make_shared(inner_type)); - ListColumnReader reader(nested_list_schema("outer", inner_type, 1, 2, 1, 2), outer_type, - std::move(inner_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(4, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 4); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 4); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 4); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 2); - EXPECT_EQ(outer_offsets[2], 5); - EXPECT_EQ(outer_offsets[3], 5); - - const auto& inner_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(inner_nullable.size(), 5); - EXPECT_FALSE(inner_nullable.is_null_at(0)); - EXPECT_FALSE(inner_nullable.is_null_at(1)); - EXPECT_TRUE(inner_nullable.is_null_at(2)); - EXPECT_FALSE(inner_nullable.is_null_at(3)); - EXPECT_TRUE(inner_nullable.is_null_at(4)); - - const auto& inner_array = assert_cast(inner_nullable.get_nested_column()); - const auto& inner_offsets = inner_array.get_offsets(); - ASSERT_EQ(inner_offsets.size(), 5); - EXPECT_EQ(inner_offsets[0], 2); - EXPECT_EQ(inner_offsets[1], 3); - EXPECT_EQ(inner_offsets[2], 3); - EXPECT_EQ(inner_offsets[3], 4); - EXPECT_EQ(inner_offsets[4], 4); - EXPECT_EQ(element_reader_ptr->build_lengths(), std::vector({4})); -} - -TEST(ParquetColumnReaderControlTest, MapKeepsEmptyMapRows) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {1}, - std::vector {0}); - auto* value_reader_ptr = value_reader.get(); - MapColumnReader reader(nested_map_schema(), nested_map_schema().type, std::move(key_reader), - std::move(value_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 1); - - const auto& nullable_map = assert_cast(*column); - EXPECT_FALSE(nullable_map.is_null_at(0)); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 0); - EXPECT_EQ(value_reader_ptr->build_lengths(), std::vector({0})); -} - -TEST(ParquetColumnReaderControlTest, ListMapSkipsAncestorEmptyRowsBeforeScalarValues) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 4, 4, 2, 4), - make_nullable(std::make_shared()), std::vector {1, 4}, - std::vector {0, 0}); - auto value_reader = make_scripted_scalar_reader(nested_int64_schema("value", 5, 5, 2, 4), - scalar_batch({1, 5}, {0, 0}, {-1, 0}, {100})); - - const auto map_type = make_nullable( - std::make_shared(make_nullable(std::make_shared()), - make_nullable(std::make_shared()))); - auto map_reader = std::make_unique( - nested_map_schema(make_nullable(std::make_shared())), map_type, - std::move(key_reader), std::move(value_reader)); - auto outer_type = make_nullable(std::make_shared(map_type)); - ListColumnReader reader(nested_list_schema("outer", map_type, 1, 2, 1, 2), outer_type, - std::move(map_reader)); - - auto column = reader.type()->create_column(); - int64_t rows_read = 0; - auto status = reader.build_nested_column(2, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(rows_read, 2); - - const auto& nullable_column = assert_cast(*column); - ASSERT_EQ(nullable_column.size(), 2); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_FALSE(nullable_column.is_null_at(1)); - - const auto& outer_array = assert_cast(nullable_column.get_nested_column()); - const auto& outer_offsets = outer_array.get_offsets(); - ASSERT_EQ(outer_offsets.size(), 2); - EXPECT_EQ(outer_offsets[0], 0); - EXPECT_EQ(outer_offsets[1], 1); - - const auto& map_nullable = assert_cast(outer_array.get_data()); - ASSERT_EQ(map_nullable.size(), 1); - EXPECT_FALSE(map_nullable.is_null_at(0)); - const auto& map_column = assert_cast(map_nullable.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 1); - - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 1); - EXPECT_FALSE(values.is_null_at(0)); - EXPECT_EQ(value_data.get_element(0), 100); -} - -TEST(ParquetColumnReaderControlTest, MapRejectsNullKeysAndMisalignedScalarValueRepLevels) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}, false, true); - auto value_reader = std::make_unique( - nested_int64_schema("value", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2}, std::vector {0}); - MapColumnReader null_key_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(value_reader)); - auto column = null_key_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = null_key_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null key"), std::string::npos); - - auto aligned_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(aligned_key_reader), - std::move(misaligned_value_reader)); - column = misaligned_reader.type()->create_column(); - status = misaligned_reader.build_nested_column(1, column, &rows_read); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapConsumePreservesKeyAndValueCorruptionChecks) { - auto null_key_reader = - make_scripted_scalar_reader(nested_int64_schema("key", 2, 3, 1, 2), - scalar_batch({2}, {0}, {-1}, std::vector {})); - auto value_reader = std::make_unique( - nested_int64_schema("value", 2, 3, 1, 2), - make_nullable(std::make_shared()), std::vector {2}, - std::vector {0}); - MapColumnReader null_key_reader_map(nested_map_schema(), nested_map_schema().type, - std::move(null_key_reader), std::move(value_reader)); - int64_t values_consumed = 0; - auto status = null_key_reader_map.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains null"), std::string::npos); - - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto misaligned_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 0}, {0, 1}, {100, 200})); - MapColumnReader misaligned_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(misaligned_value_reader)); - status = misaligned_reader.consume_nested_column(1, &values_consumed); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("value repetition level is not aligned"), std::string::npos); -} - -TEST(ParquetColumnReaderControlTest, MapBuildsScalarAndComplexValuePaths) { - auto key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto scalar_value_reader = - make_scripted_scalar_reader(nested_int64_schema("value", 2, 3, 1), - scalar_batch({3, 3}, {0, 1}, {0, 1}, {100, 200})); - MapColumnReader scalar_reader(nested_map_schema(), nested_map_schema().type, - std::move(key_reader), std::move(scalar_value_reader)); - auto column = scalar_reader.type()->create_column(); - int64_t rows_read = 0; - auto status = scalar_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - const auto& nullable_map = assert_cast(*column); - const auto& map_column = assert_cast(nullable_map.get_nested_column()); - ASSERT_EQ(map_column.get_offsets().size(), 1); - EXPECT_EQ(map_column.get_offsets()[0], 2); - const auto& values = assert_cast(map_column.get_values()); - const auto& value_data = assert_cast(values.get_nested_column()); - ASSERT_EQ(values.size(), 2); - EXPECT_EQ(value_data.get_element(0), 100); - EXPECT_EQ(value_data.get_element(1), 200); - - auto complex_key_reader = std::make_unique( - nested_int64_schema("key", 1, 2, 1), make_nullable(std::make_shared()), - std::vector {2, 2}, std::vector {0, 1}); - auto complex_value_reader = std::make_unique( - nested_int64_schema("complex_value", 2, 3, 1), - make_nullable(std::make_shared()), std::vector {3, 3}, - std::vector {0, 1}); - auto* complex_value_reader_ptr = complex_value_reader.get(); - MapColumnReader complex_reader(nested_map_schema(), nested_map_schema().type, - std::move(complex_key_reader), std::move(complex_value_reader)); - column = complex_reader.type()->create_column(); - status = complex_reader.build_nested_column(1, column, &rows_read); - ASSERT_TRUE(status.ok()) << status; - EXPECT_EQ(complex_value_reader_ptr->build_lengths(), std::vector({2})); -} - TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { RowPositionColumnReader reader(100); EXPECT_EQ(reader.file_column_id(), format::ROW_POSITION_COLUMN_ID); @@ -1271,79 +237,4 @@ TEST(ParquetVirtualColumnReaderTest, GlobalRowIdReadSkipSelectAndInvalidArgs) { EXPECT_FALSE(reader.read(1, column, nullptr).ok()); } -TEST(ParquetColumnReaderFactoryTest, RejectsInvalidLeafIdBeforeCreatingRecordReader) { - ParquetColumnSchema schema = int64_schema("bad_leaf"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.leaf_column_id = 3; - schema.type_descriptor.physical_type = ::parquet::Type::INT64; - schema.type_descriptor.doris_type = schema.type; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Invalid parquet leaf column id"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsProjectedUnsupportedLogicalType) { - ParquetColumnSchema schema = int64_schema("unsupported_time"); - schema.kind = ParquetColumnSchemaKind::PRIMITIVE; - schema.type_descriptor.unsupported_reason = - "Parquet TIME with isAdjustedToUTC=true is not supported"; - - ParquetColumnReaderFactory factory(nullptr, 1); - std::unique_ptr reader; - const auto status = factory.create(schema, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find(schema.type_descriptor.unsupported_reason), - std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsStructInvalidAndEmptyProjection) { - auto schema = struct_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto empty_projection = format::LocalColumnIndex::partial_local(0); - status = factory.create(schema, &empty_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no children"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsListProjectionWithoutElement) { - auto schema = list_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto projection = format::LocalColumnIndex::partial_local(0); - const auto status = factory.create(schema, &projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no element"), std::string::npos); -} - -TEST(ParquetColumnReaderFactoryTest, RejectsMapInvalidAndKeyOnlyProjection) { - auto schema = map_schema_for_projection(); - ParquetColumnReaderFactory factory(nullptr, 0); - std::unique_ptr reader; - - auto invalid_projection = format::LocalColumnIndex::partial_local(0); - invalid_projection.children.push_back(format::LocalColumnIndex::local(1)); - invalid_projection.children.push_back(format::LocalColumnIndex::local(9)); - auto status = factory.create(schema, &invalid_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("invalid child"), std::string::npos); - - auto key_only_projection = format::LocalColumnIndex::partial_local(0); - key_only_projection.children.push_back(format::LocalColumnIndex::local(0)); - status = factory.create(schema, &key_only_projection, &reader); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("contains no value"), std::string::npos); -} - } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp b/be/test/format_v2/parquet/parquet_serde_reader_test.cpp deleted file mode 100644 index c35138e3263723..00000000000000 --- a/be/test/format_v2/parquet/parquet_serde_reader_test.cpp +++ /dev/null @@ -1,459 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "core/assert_cast.h" -#include "core/column/column_decimal.h" -#include "core/column/column_nullable.h" -#include "core/column/column_string.h" -#include "core/column/column_vector.h" -#include "core/data_type/data_type.h" -#include "core/data_type/data_type_nullable.h" -#include "core/types.h" -#include "format_v2/parquet/parquet_column_schema.h" -#include "format_v2/parquet/reader/column_reader.h" - -namespace doris::format::parquet { -namespace { - -constexpr int64_t ROW_COUNT = 5; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -class ParquetSerdeReaderTest : public testing::Test { -protected: - void SetUp() override { - _test_dir = std::filesystem::temp_directory_path() / "doris_parquet_serde_reader_test"; - std::filesystem::remove_all(_test_dir); - std::filesystem::create_directories(_test_dir); - _file_path = (_test_dir / "serde.parquet").string(); - write_parquet_file(); - open_file(_file_path); - } - - void TearDown() override { std::filesystem::remove_all(_test_dir); } - - template - std::shared_ptr build_required_array(const std::vector& values) { - Builder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_nullable_int32_array() { - arrow::Int32Builder builder; - EXPECT_TRUE(builder.Append(1).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(3).ok()); - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(5).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_nullable_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.AppendNull().ok()); - EXPECT_TRUE(builder.Append(0x0000).ok()); - EXPECT_TRUE(builder.Append(0x8000).ok()); - EXPECT_TRUE(builder.Append(0x3E00).ok()); - EXPECT_TRUE(builder.Append(0x7E00).ok()); - return finish_array(&builder); - } - - std::shared_ptr build_binary_array(const std::vector& values) { - arrow::BinaryBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data()), - static_cast(value.size())) - .ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_fixed_binary_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::FixedSizeBinaryBuilder builder(type, arrow::default_memory_pool()); - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(reinterpret_cast(value.data())).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_timestamp_array( - const std::shared_ptr& type, const std::vector& values) { - arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); - } - - std::shared_ptr build_decimal_array(const std::shared_ptr& type, - const std::vector& values) { - arrow::Decimal128Builder builder(type, arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(arrow::Decimal128(value)).ok()); - } - return finish_array(&builder); - } - - void add_field(const std::shared_ptr& field, - std::shared_ptr array) { - _arrow_fields.push_back(field); - _arrays.push_back(std::move(array)); - } - - void write_table(const std::string& file_path, const std::shared_ptr& table, - std::shared_ptr<::parquet::ArrowWriterProperties> arrow_properties = nullptr) { - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - ::parquet::WriterProperties::Builder writer_builder; - writer_builder.version(::parquet::ParquetVersion::PARQUET_2_6); - writer_builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - writer_builder.compression(::parquet::Compression::UNCOMPRESSED); - if (arrow_properties == nullptr) { - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_properties = arrow_builder.build(); - } - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable( - *table, arrow::default_memory_pool(), *file_result, ROW_COUNT, - writer_builder.build(), std::move(arrow_properties))); - } - - void write_parquet_file() { - add_field(arrow::field("bool_col", arrow::boolean(), false), - build_required_array( - {true, false, true, false, true})); - add_field(arrow::field("int32_col", arrow::int32(), false), - build_required_array({10, 20, 30, 40, 50})); - add_field(arrow::field("int64_col", arrow::int64(), false), - build_required_array( - {10000000000L, -9L, 42L, 77L, 123L})); - add_field(arrow::field("uint32_col", arrow::uint32(), false), - build_required_array( - {0U, 1U, 1U << 31, std::numeric_limits::max(), 42U})); - add_field(arrow::field("uint64_col", arrow::uint64(), false), - build_required_array( - {0ULL, 1ULL, 1ULL << 63, std::numeric_limits::max(), 42ULL})); - add_field(arrow::field("float_col", arrow::float32(), false), - build_required_array( - {1.5F, -2.25F, 3.0F, 4.5F, 5.75F})); - add_field(arrow::field("double_col", arrow::float64(), false), - build_required_array({3.5, -4.75, 6.0, 7.25, 8.5})); - add_field(arrow::field("nullable_float16_col", arrow::float16(), true), - build_nullable_float16_array()); - add_field(arrow::field("binary_col", arrow::binary(), false), - build_binary_array({"bin_a", "bin_b", "bin_c", "bin_d", "bin_e"})); - add_field(arrow::field("string_col", arrow::utf8(), false), - build_string_array({"alpha", "beta", "gamma", "delta", "epsilon"})); - add_field(arrow::field("fixed_binary_col", arrow::fixed_size_binary(4), false), - build_fixed_binary_array(arrow::fixed_size_binary(4), - {"aaaa", "bbbb", "cccc", "dddd", "eeee"})); - add_field(arrow::field("date_col", arrow::date32(), false), - build_required_array({0, 1, 18628, 18629, 18630})); - add_field(arrow::field("timestamp_millis_col", arrow::timestamp(arrow::TimeUnit::MILLI), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MILLI), - {0, 1234, 1609459200000, 1609459201000, -1})); - add_field(arrow::field("timestamp_micros_col", arrow::timestamp(arrow::TimeUnit::MICRO), - false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("timestamp_micros_utc_col", - arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - {0, 1234567, 1609459200000000, 1609459201000000, -1})); - add_field(arrow::field("decimal_fixed_binary_9_2_col", arrow::decimal128(9, 2), false), - build_decimal_array(arrow::decimal128(9, 2), {12345, -67, 0, 987, 1000})); - add_field(arrow::field("decimal_fixed_binary_18_6_col", arrow::decimal128(18, 6), false), - build_decimal_array(arrow::decimal128(18, 6), - {1234567, -670000, 0, 9870000, 1000000})); - add_field(arrow::field("nullable_int_col", arrow::int32(), true), - build_nullable_int32_array()); - - write_table(_file_path, arrow::Table::Make(arrow::schema(_arrow_fields), _arrays)); - } - - void open_file(const std::string& file_path) { - _file_reader = ::parquet::ParquetFileReader::OpenFile(file_path, false); - ASSERT_NE(_file_reader, nullptr); - ASSERT_EQ(_file_reader->metadata()->num_row_groups(), 1); - _row_group = _file_reader->RowGroup(0); - ASSERT_NE(_row_group, nullptr); - auto schema_descriptor = _file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - auto st = build_parquet_column_schema(*schema_descriptor, &_fields); - ASSERT_TRUE(st.ok()) << st; - } - - size_t find_field_idx(const std::string& name) const { - for (size_t field_idx = 0; field_idx < _fields.size(); ++field_idx) { - if (_fields[field_idx]->name == name) { - return field_idx; - } - } - ADD_FAILURE() << "Cannot find parquet serde test field " << name; - return _fields.size(); - } - - std::unique_ptr create_reader(size_t field_idx) const { - ParquetColumnReaderFactory factory(_row_group, _file_reader->metadata()->num_columns()); - std::unique_ptr reader; - auto st = factory.create(*_fields[field_idx], &reader); - EXPECT_TRUE(st.ok()) << st; - return reader; - } - - template - void read_and_validate(const std::string& name, Validator validator) const { - const auto field_idx = find_field_idx(name); - ASSERT_TRUE(supports_record_reader(_fields[field_idx]->type_descriptor)); - auto reader = create_reader(field_idx); - ASSERT_NE(reader, nullptr); - MutableColumnPtr column = reader->type()->create_column(); - int64_t rows_read = 0; - auto st = reader->read(ROW_COUNT, column, &rows_read); - ASSERT_TRUE(st.ok()) << st; - ASSERT_EQ(rows_read, ROW_COUNT); - ASSERT_EQ(column->size(), ROW_COUNT); - validator(*_fields[field_idx], *column); - } - - std::filesystem::path _test_dir; - std::string _file_path; - std::unique_ptr<::parquet::ParquetFileReader> _file_reader; - std::shared_ptr<::parquet::RowGroupReader> _row_group; - std::vector> _fields; - std::vector> _arrow_fields; - std::vector> _arrays; -}; - -TEST_F(ParquetSerdeReaderTest, ReadAllSupportedPhysicalAndLogicalTypes) { - read_and_validate("bool_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BOOLEAN); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 1); - EXPECT_EQ(values.get_element(1), 0); - EXPECT_EQ(values.get_element(4), 1); - }); - read_and_validate("int32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10); - EXPECT_EQ(values.get_element(4), 50); - }); - read_and_validate("int64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), 10000000000L); - EXPECT_EQ(values.get_element(1), -9L); - }); - read_and_validate("uint32_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_BIGINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), 2147483648L); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("uint64_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.is_unsigned_integer); - EXPECT_EQ(schema.type_descriptor.integer_bit_width, 64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_LARGEINT); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(2), static_cast(1) << 63); - EXPECT_EQ(values.get_element(3), - static_cast(std::numeric_limits::max())); - }); - read_and_validate("float_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FLOAT); - const auto& values = assert_cast(column); - EXPECT_FLOAT_EQ(values.get_element(0), 1.5F); - EXPECT_FLOAT_EQ(values.get_element(1), -2.25F); - }); - read_and_validate("double_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::DOUBLE); - const auto& values = assert_cast(column); - EXPECT_DOUBLE_EQ(values.get_element(0), 3.5); - EXPECT_DOUBLE_EQ(values.get_element(1), -4.75); - }); - read_and_validate("nullable_float16_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 2); - EXPECT_EQ(schema.type_descriptor.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(schema.type_descriptor.is_string_like); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_FLOAT); - const auto& nullable_column = assert_cast(column); - const auto& values = assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_TRUE(nullable_column.is_null_at(0)); - EXPECT_FLOAT_EQ(values.get_element(1), 0.0F); - EXPECT_FALSE(std::signbit(values.get_element(1))); - EXPECT_FLOAT_EQ(values.get_element(2), -0.0F); - EXPECT_TRUE(std::signbit(values.get_element(2))); - EXPECT_FLOAT_EQ(values.get_element(3), 1.5F); - EXPECT_TRUE(std::isnan(values.get_element(4))); - }); - read_and_validate("binary_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::BYTE_ARRAY); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "bin_a"); - EXPECT_EQ(values.get_data_at(3).to_string(), "bin_d"); - }); - read_and_validate("string_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type_descriptor.is_string_like); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "alpha"); - EXPECT_EQ(values.get_data_at(4).to_string(), "epsilon"); - }); - read_and_validate("fixed_binary_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(schema.type_descriptor.fixed_length, 4); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_data_at(0).to_string(), "aaaa"); - EXPECT_EQ(values.get_data_at(2).to_string(), "cccc"); - }); - read_and_validate("date_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT32); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATEV2); - EXPECT_EQ(schema.type->to_string(column, 0), "1970-01-01"); - EXPECT_EQ(schema.type->to_string(column, 2), "2021-01-01"); - }); - read_and_validate( - "timestamp_millis_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999"); - }); - read_and_validate( - "timestamp_micros_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("timestamp_micros_utc_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::INT64); - EXPECT_TRUE(schema.type_descriptor.timestamp_is_adjusted_to_utc); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(schema.type->to_string(column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(schema.type->to_string(column, 4), "1969-12-31 23:59:59.999999"); - }); - read_and_validate("decimal_fixed_binary_9_2_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL32); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal32(12345)); - EXPECT_EQ(schema.type->to_string(column, 0), "123.45"); - }); - read_and_validate("decimal_fixed_binary_18_6_col", [](const ParquetColumnSchema& schema, - const IColumn& column) { - EXPECT_EQ(schema.type_descriptor.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_TRUE(schema.type_descriptor.is_decimal); - EXPECT_EQ(remove_nullable(schema.type)->get_primitive_type(), TYPE_DECIMAL64); - const auto& values = assert_cast(column); - EXPECT_EQ(values.get_element(0), Decimal64(1234567)); - EXPECT_EQ(schema.type->to_string(column, 0), "1.234567"); - }); - read_and_validate( - "nullable_int_col", [](const ParquetColumnSchema& schema, const IColumn& column) { - EXPECT_TRUE(schema.type->is_nullable()); - const auto& nullable_column = assert_cast(column); - const auto& nested_column = - assert_cast(nullable_column.get_nested_column()); - ASSERT_EQ(nullable_column.size(), ROW_COUNT); - EXPECT_FALSE(nullable_column.is_null_at(0)); - EXPECT_TRUE(nullable_column.is_null_at(1)); - EXPECT_FALSE(nullable_column.is_null_at(2)); - EXPECT_TRUE(nullable_column.is_null_at(3)); - EXPECT_EQ(nested_column.get_element(0), 1); - EXPECT_EQ(nested_column.get_element(2), 3); - }); -} - -TEST_F(ParquetSerdeReaderTest, ReadInt96TimestampAsDateTimeV2) { - const auto file_path = (_test_dir / "int96_timestamp.parquet").string(); - auto field = arrow::field("col_datetime", arrow::timestamp(arrow::TimeUnit::MICRO), false); - auto array = build_timestamp_array(arrow::timestamp(arrow::TimeUnit::MICRO), - {0, 1234567, 1609459200000000, 1609459201000000, -1}); - auto table = arrow::Table::Make(arrow::schema({field}), {array}); - - ::parquet::ArrowWriterProperties::Builder arrow_builder; - arrow_builder.enable_force_write_int96_timestamps(); - _fields.clear(); - _file_reader.reset(); - _row_group.reset(); - write_table(file_path, table, arrow_builder.build()); - open_file(file_path); - - ASSERT_EQ(_fields.size(), 1); - EXPECT_EQ(_fields[0]->type_descriptor.physical_type, ::parquet::Type::INT96); - EXPECT_EQ(_fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - ASSERT_TRUE(supports_record_reader(_fields[0]->type_descriptor)); - ASSERT_EQ(remove_nullable(_fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - - auto reader = create_reader(0); - ASSERT_NE(reader, nullptr); - auto column = _fields[0]->type->create_column(); - int64_t rows_read = 0; - ASSERT_TRUE(reader->read(ROW_COUNT, column, &rows_read).ok()); - ASSERT_EQ(rows_read, ROW_COUNT); - EXPECT_EQ(_fields[0]->type->to_string(*column, 0), "1970-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 1), "1970-01-01 00:00:01.234567"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 2), "2021-01-01 00:00:00.000000"); - EXPECT_EQ(_fields[0]->type->to_string(*column, 4), "1969-12-31 23:59:59.999999"); -} - -} // namespace -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp index 4bca77c1803b49..ecc4fad53ccaf9 100644 --- a/be/test/format_v2/parquet/parquet_type_test.cpp +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -103,7 +103,6 @@ TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); EXPECT_EQ(type.integer_bit_width, test_case.bit_width); EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_TRUE(type.supports_record_reader); } } @@ -131,7 +130,6 @@ TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), ::parquet::Type::INT32)); EXPECT_EQ(adjusted_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_time.supports_record_reader); EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( @@ -193,7 +191,6 @@ TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, ::parquet::ConvertedType::TIME_MILLIS)); EXPECT_EQ(converted_time.doris_type, nullptr); - EXPECT_FALSE(converted_time.supports_record_reader); EXPECT_FALSE(converted_time.unsupported_reason.empty()); const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( @@ -410,7 +407,6 @@ TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { const auto null_type = resolve_parquet_type(nullptr); EXPECT_EQ(null_type.doris_type, nullptr); EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); - EXPECT_TRUE(null_type.supports_record_reader); const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); @@ -462,7 +458,6 @@ TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); EXPECT_EQ(type.is_string_like, test_case.expected_string_like); - EXPECT_TRUE(type.supports_record_reader); } } @@ -482,7 +477,6 @@ TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), ::parquet::Type::INT64)); EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_nanos_time.supports_record_reader); EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 72b5e766052b61..92fd83eb26f90e 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -447,55 +447,38 @@ batches. #### Complex-reader interface and materialization cost -The retained Arrow complex-reader facade exposes four stateful operations: -`load_nested_batch()`, `load_nested_levels_batch()`, `build_nested_column()`, and -`consume_nested_column()`. This split made shape-only reads possible while Arrow still owned value -decoding, but it is not the target native interface. It has three measurable costs: - -1. callers must preserve an implicit load-before-build/consume phase and multiple nested cursors; -2. ARRAY/MAP/STRUCT layers can rescan the same def/rep span to derive parent boundaries, child - counts, null maps, and alignment; -3. a materialized leaf can copy levels, build a level-to-payload index, build a temporary null map, - and only then convert the payload to a Doris column. - -Doris v1 is simpler at the public boundary: a child `read_column_data()` call owns physical decode -and exposes its persistent def/rep buffers to the collection reader. It nevertheless repeats some -level interpretation in collection helpers, so v1 is a compatibility/performance baseline rather -than the final abstraction. DuckDB uses a single `Read(input, vector)` operation. Its LIST reader -reads a reusable child vector plus def/rep buffers, emits list entries in the same traversal, and -keeps overflow for the next vector; its STRUCT reader invokes children directly into output vectors -and verifies their row counts. See DuckDB's official -[LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), +The original v2 prototype placed an Arrow-decoded leaf batch and separate load/build/consume phases +between encoded pages and Doris columns. That design required implicit phase ordering, duplicated +level traversal in ARRAY/MAP/STRUCT wrappers, and retained decoded binary payload even when a caller +needed only nullability. The prototype hierarchy and its batch container have been removed. + +The production boundary is now the same compact cursor contract used for scalar columns: +`NativeColumnReader::read/select/skip`. Its persistent native reader owns page, decoder, level, +conversion, and complex-column state and materializes the complete result directly into the caller's +Doris column. No Arrow value reader, intermediate decoded leaf container, temporary nested Doris +column, or public load-before-build protocol participates in predicate or output scans. + +Internally, complex decoding still has to solve the shared-level-plan problem. For example, levels +representing `[["a", "b"], NULL, []]` produce entry counts `[2, 0, 0]`, parent nulls `[0, 1, 0]`, +and string payload ordinals `[0, 1]`. MAP uses the key leaf as the entry-shape owner and validates +the value leaf against it; STRUCT children advance in parent-row lockstep. This state belongs behind +the native reader boundary rather than in caller-visible phases. + +`CountColumnReader` is the only data-page compatibility exception. Existing +`COUNT(nullable_col)` pushdown needs definition/repetition levels but Arrow does not expose its +level decoder independently of `RecordReader`. The adapter therefore selects one representative +leaf (the key for MAP), calls `ReadRecords`, copies only levels, and immediately releases any binary +builder chunks. It exposes neither decoded values nor the ordinary scan-reader API, so it cannot +become an Arrow fallback. This preserves the existing pushdown semantics while keeping large complex +values out of the retained aggregate state. + +Doris v1 remains the behavior/performance baseline: `read_column_data()` owns physical decode and +the collection reader consumes persistent level buffers. DuckDB provides the same useful design +principle through its single `Read(input, vector)` boundary and reusable child vectors. See DuckDB's +official [LIST reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/list_column_reader.cpp), [STRUCT reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/reader/struct_column_reader.cpp), and [base column reader](https://github.com/duckdb/duckdb/blob/main/extension/parquet/column_reader.cpp). -The native v2 boundary therefore uses one operation conceptually equivalent to: - -```text -read_nested(request { parent_rows, selection, VALUES | LEVELS_ONLY }, output) - -> result { parent_rows, shared_level_plan, payload_counts } -``` - -`VALUES` decodes selected payload and materializes directly into the supplied Doris child columns; -`LEVELS_ONLY` advances identical level/page cursors without materializing payload. Both modes build -the same parent boundaries and validation decisions. ARRAY, MAP, and STRUCT consume a shared plan -rather than calling separate public build/consume phases. For example, levels representing -`[["a", "b"], NULL, []]` produce parent boundaries `[0, 2, 3, 4]`, entry counts `[2, 0, 0]`, and -parent nulls `[0, 1, 0]` in one traversal; string payload ordinals are `[0, 1]`. MAP uses the key -leaf as the entry-shape owner and validates the value leaf against the same entry plan. STRUCT uses -one representative leaf for parent validity and only checks sibling alignment while decoding each -child. - -Ordinary production scans do not call these four methods: `NativeColumnReader::read/select/skip` -uses the native `read_column_data()` boundary and materializes the complete complex column directly. -The facade remains for the unchanged levels-only aggregate path and focused control tests; it is not -a fallback after native reader selection. `ParquetLeafBatch` is the facade's decoded Arrow/level -container and is intentionally forbidden from the ordinary value path. Its leaf reader, -SerDe, binary/null/level scratch, selection ranges, nested batches, parent nulls, entry counts, and -child-column handles must be persistent so the facade does not add per-batch allocation churn. -New native decoder code must not depend on this phase ordering or reproduce the temporary -level-to-payload index when it can stream payload positions directly from the shared plan. - ### 7.5 Index Coordinate Domains and Composition Index correctness depends on keeping its coordinate systems separate: From 01ea50c879e55fcba65ed5509581a17a8e1568c6 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 19:33:40 +0800 Subject: [PATCH 05/34] [improvement](be) Materialize Parquet v2 values through SerDe ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: FileScannerV2 still paid for an Arrow-decoded value layer or decoder-owned logical conversion before constructing Doris columns. This duplicated buffers for strings and nested values, amplified adaptive-batch memory peaks, and made dictionary, page-cache, and decode profiles difficult to compare with the v1 reader. Introduce an independent Parquet page/encoding reader under format_v2 whose decoders only parse encoded streams and own cursors. DataTypeSerDe now interprets Parquet physical and logical annotations and materializes selected values directly into persistent Doris columns. The native reader supports the existing scalar, decimal, date/time, timestamp, UUID, dictionary, delta, byte-stream-split, Page V1/V2, nested level, selection, page-index, footer/page-cache, and MergeRange paths. COUNT complex columns use a levels-only reader with persistent scratch, and page/decode/cache profiles expose cumulative deltas without duplicate counting. Arrow remains only in metadata/index planning and test fixture generation. No FE code or v1 format decoder is modified. ### Release note Improve FileScannerV2 Parquet scan memory use and native decode performance, especially for strings, decimals, nested columns, dictionary selection, and adaptive batch sizes. ### Check List (For Author) - Test: Unit Test - 14 Parquet decoder and DataTypeSerDe tests on the designated remote host - 76 Parquet v2 reader, selection, page-index, complex-column, and scan tests on the designated remote host - 6 focused COUNT/profile tests on the designated remote host - Behavior changed: Yes (FileScannerV2 Parquet data pages materialize through the native v2 decoder and DataTypeSerDe; query semantics are unchanged) - Does this need documentation: Yes (updated the FileScannerV2 Parquet design and review guide) --- .../data_type_datetimev2_serde.cpp | 94 ++ .../data_type_datetimev2_serde.h | 5 + .../data_type_datev2_serde.cpp | 70 +- .../data_type_serde/data_type_datev2_serde.h | 5 + .../data_type_decimal_serde.cpp | 174 ++- .../data_type_serde/data_type_decimal_serde.h | 5 + .../data_type_number_serde.cpp | 206 +++ .../data_type_serde/data_type_number_serde.h | 5 + .../core/data_type_serde/data_type_serde.cpp | 13 + be/src/core/data_type_serde/data_type_serde.h | 15 + .../data_type_string_serde.cpp | 91 ++ .../data_type_serde/data_type_string_serde.h | 5 + .../data_type_serde/data_type_time_serde.cpp | 85 ++ .../data_type_serde/data_type_time_serde.h | 5 + .../data_type_timestamptz_serde.cpp | 73 + .../data_type_timestamptz_serde.h | 5 + .../data_type_varbinary_serde.cpp | 63 + .../data_type_varbinary_serde.h | 6 + .../data_type_serde/parquet_decode_source.h | 151 ++ be/src/format_v2/AGENTS.md | 51 +- be/src/format_v2/parquet/parquet_profile.cpp | 12 +- be/src/format_v2/parquet/parquet_profile.h | 20 +- be/src/format_v2/parquet/parquet_reader.cpp | 25 +- be/src/format_v2/parquet/parquet_scan.cpp | 12 +- .../format_v2/parquet/reader/column_reader.h | 8 +- .../parquet/reader/count_column_reader.cpp | 275 ++-- .../parquet/reader/count_column_reader.h | 43 +- .../reader/native/bool_plain_decoder.cpp | 75 + .../reader/native/bool_plain_decoder.h | 91 ++ .../reader/native/bool_rle_decoder.cpp | 67 + .../parquet/reader/native/bool_rle_decoder.h | 53 + .../reader/native/byte_array_dict_decoder.cpp | 63 + .../reader/native/byte_array_dict_decoder.h | 45 + .../native/byte_array_plain_decoder.cpp | 67 + .../reader/native/byte_array_plain_decoder.h | 56 + .../native/byte_stream_split_decoder.cpp | 51 + .../reader/native/byte_stream_split_decoder.h | 38 + .../reader/native/column_chunk_reader.cpp | 867 +++++++++++ .../reader/native/column_chunk_reader.h | 290 ++++ .../parquet/reader/native/column_reader.cpp | 1340 +++++++++++++++++ .../parquet/reader/native/column_reader.h | 570 +++++++ .../parquet/reader/native/decoder.cpp | 152 ++ .../format_v2/parquet/reader/native/decoder.h | 143 ++ .../reader/native/delta_bit_pack_decoder.cpp | 133 ++ .../reader/native/delta_bit_pack_decoder.h | 438 ++++++ .../reader/native/fix_length_dict_decoder.hpp | 64 + .../native/fix_length_plain_decoder.cpp | 42 + .../reader/native/fix_length_plain_decoder.h | 44 + .../parquet/reader/native/level_decoder.cpp | 107 ++ .../parquet/reader/native/level_decoder.h | 69 + .../parquet/reader/native/level_reader.cpp | 205 +++ .../parquet/reader/native/level_reader.h | 74 + .../parquet/reader/native/page_reader.cpp | 220 +++ .../parquet/reader/native/page_reader.h | 261 ++++ .../parquet/reader/native_column_reader.cpp | 47 +- .../parquet/reader/native_column_reader.h | 13 +- .../data_type_serde_parquet_test.cpp | 322 ++++ .../format_v2/parquet/native_decoder_test.cpp | 253 ++++ .../parquet/parquet_reader_control_test.cpp | 2 +- .../format_v2/parquet/parquet_reader_test.cpp | 20 +- .../format_v2/parquet/parquet_scan_test.cpp | 4 +- docs/file-scanner-v2-code-review-guide.md | 30 + docs/file-scanner-v2-parquet-scan-design.md | 105 +- 63 files changed, 7597 insertions(+), 346 deletions(-) create mode 100644 be/src/core/data_type_serde/parquet_decode_source.h create mode 100644 be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/bool_plain_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/bool_rle_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/column_chunk_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/column_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/column_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp create mode 100644 be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/level_decoder.cpp create mode 100644 be/src/format_v2/parquet/reader/native/level_decoder.h create mode 100644 be/src/format_v2/parquet/reader/native/level_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/level_reader.h create mode 100644 be/src/format_v2/parquet/reader/native/page_reader.cpp create mode 100644 be/src/format_v2/parquet/reader/native/page_reader.h create mode 100644 be/test/core/data_type_serde/data_type_serde_parquet_test.cpp create mode 100644 be/test/format_v2/parquet/native_decoder_test.cpp diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 44402bb5b531cd..ad22905c4acb0a 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -31,10 +31,12 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" #include "exprs/function/cast/cast_to_string.h" +#include "util/unaligned.h" enum { DIVISOR_FOR_SECOND = 1, @@ -133,6 +135,64 @@ int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { return value; } +int64_t parquet_timestamp_micros(const ParquetDecodeContext& context, int64_t value) { + if (context.time_unit == ParquetTimeUnit::MILLIS) { + return value * 1000; + } + if (context.time_unit == ParquetTimeUnit::NANOS) { + return value / 1000; + } + return value; +} + +class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context) + : _data(assert_cast(column).get_data()), _context(context) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + static const auto utc_timezone = cctz::utc_time_zone(); + const auto& timezone = _context.timezone == nullptr ? utc_timezone : *_context.timezone; + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(DecodedInt96Timestamp)); + timestamp_micros = unaligned_load( + values + row * sizeof(DecodedInt96Timestamp)) + .to_timestamp_micros(); + append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, timezone); + continue; + } + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + timestamp_micros = parquet_timestamp_micros( + _context, unaligned_load(values + row * sizeof(int64_t))); + if (_context.timestamp_is_adjusted_to_utc) { + append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, timezone); + continue; + } + auto status = append_datetimev2_from_epoch_micros(_data, timestamp_micros); + if (!status.ok()) { + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + const ParquetDecodeContext& _context; +}; + +class RejectDateTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATETIMEV2"); + } +}; + } // namespace // NOLINTBEGIN(readability-function-size) @@ -599,6 +659,40 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( return Status::OK(); } +Status DataTypeDateTimeV2SerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateTimeV2ParquetConsumer consumer(column, context); + RejectDateTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("DATETIMEV2 expects Parquet INT64 or INT96"); + } + DateTimeV2ParquetConsumer consumer(column, context); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} + Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.h b/be/src/core/data_type_serde/data_type_datetimev2_serde.h index e089d789ccee7d..c3411ed4249843 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.h @@ -90,6 +90,11 @@ class DataTypeDateTimeV2SerDe : public DataTypeNumberSerDe(column).get_data()) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + DateV2Value value; + value.get_date_from_daynr(unaligned_load(values + row * sizeof(int32_t)) + + date_threshold); + _data[old_size + row] = value; + } + return Status::OK(); + } + +private: + ColumnDateV2::Container& _data; +}; + +class RejectDateV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as DATEV2"); + } +}; + +} // namespace Status DataTypeDateV2SerDe::serialize_column_to_json(const IColumn& column, int64_t start_idx, int64_t end_idx, BufferWritable& bw, @@ -153,6 +187,38 @@ Status DataTypeDateV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeDateV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + DateV2ParquetConsumer consumer(column); + RejectDateV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT32 || + context.logical_type != ParquetLogicalType::DATE) { + return Status::NotSupported("DATEV2 expects Parquet DATE stored as INT32"); + } + DateV2ParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} + Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.h b/be/src/core/data_type_serde/data_type_datev2_serde.h index bc02b61b520193..39d44efcfa0652 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.h +++ b/be/src/core/data_type_serde/data_type_datev2_serde.h @@ -88,6 +88,11 @@ class DataTypeDateV2SerDe : public DataTypeNumberSerDe bool decoded_decimal_value_fits(const typename PrimitiveTypeTraits::CppType::NativeType& value, UInt32 precision) { - return value >= min_decimal_value(precision).value && - value <= max_decimal_value(precision).value; + if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; + } else { + return value >= min_decimal_value(precision).value && + value <= max_decimal_value(precision).value; + } } template @@ -85,6 +91,9 @@ bool decoded_decimal_int_value_fits(Int128 value, UInt32 precision) { if constexpr (std::is_same_v) { const auto wide_value = wide::Int256(value); return decoded_decimal_value_fits(wide_value, precision); + } else if constexpr (T == TYPE_DECIMALV2) { + const auto limit = DataTypeDecimal::get_max_digits_number(precision); + return value >= -limit && value <= limit; } else { return value >= static_cast(min_decimal_value(precision).value) && value <= static_cast(max_decimal_value(precision).value); @@ -179,6 +188,125 @@ Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& vie return Status::OK(); } +template +Status scale_parquet_decimal(typename PrimitiveTypeTraits::CppType::NativeType value, + int32_t source_scale, int32_t target_scale, + typename PrimitiveTypeTraits::CppType::NativeType* result) { + using NativeType = typename PrimitiveTypeTraits::CppType::NativeType; + DORIS_CHECK(result != nullptr); + if (source_scale == target_scale) { + *result = value; + return Status::OK(); + } + if (source_scale > target_scale) { + *result = value / decimal_scale_multiplier(source_scale - target_scale); + return Status::OK(); + } + const auto multiplier = decimal_scale_multiplier(target_scale - source_scale); + if (common::mul_overflow(value, multiplier, *result)) { + return Status::DataQualityError("Parquet decimal overflows while scaling from {} to {}", + source_scale, target_scale); + } + return Status::OK(); +} + +template +class DecimalParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + using FieldType = typename PrimitiveTypeTraits::CppType; + using NativeType = typename FieldType::NativeType; + + DecimalParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + UInt32 target_precision, int32_t target_scale) + : _data(assert_cast&>(column).get_data()), + _context(context), + _target_precision(target_precision), + _target_scale(target_scale) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_integers(values, num_values); + } + if (_context.physical_type == ParquetPhysicalType::INT64) { + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_integers(values, num_values); + } + if (_context.physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return Status::NotSupported("Unsupported Parquet physical type {} for decimal SerDe", + static_cast(_context.physical_type)); + } + DORIS_CHECK_EQ(value_width, static_cast(_context.type_length)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = + append_binary_value(values + row * value_width, value_width, old_size + row); + if (!status.ok()) { + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + auto status = append_binary_value(reinterpret_cast(values[row].data), + values[row].size, old_size + row); + if (!status.ok()) { + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + template + Status append_integers(const uint8_t* values, size_t num_values) { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = unaligned_load(values + row * sizeof(SourceType)); + auto status = append_native_value(NativeType(source_value), old_size + row); + if (!status.ok()) { + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + + Status append_binary_value(const uint8_t* value, size_t length, size_t output_row) { + if (UNLIKELY(length > sizeof(NativeType))) { + return Status::DataQualityError("Parquet decimal binary value is too wide: {}", length); + } + return append_native_value( + decode_big_endian_signed_integer(value, cast_set(length)), + output_row); + } + + Status append_native_value(NativeType value, size_t output_row) { + NativeType scaled_value; + RETURN_IF_ERROR(scale_parquet_decimal(value, _context.decimal_scale, _target_scale, + &scaled_value)); + if (!decoded_decimal_value_fits(scaled_value, _target_precision)) { + return Status::DataQualityError("Parquet decimal value is out of range"); + } + _data[output_row] = FieldType {scaled_value}; + return Status::OK(); + } + + typename ColumnDecimal::Container& _data; + const ParquetDecodeContext& _context; + UInt32 _target_precision; + int32_t _target_scale; +}; + } // namespace template @@ -529,6 +657,48 @@ Status DataTypeDecimalSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeDecimalSerDe::read_parquet_dictionary(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { + return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); + } + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} + template Status DataTypeDecimalSerDe::write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.h b/be/src/core/data_type_serde/data_type_decimal_serde.h index 547488c9da4bc5..2a399ce83cc1a4 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.h +++ b/be/src/core/data_type_serde/data_type_decimal_serde.h @@ -109,6 +109,11 @@ class DataTypeDecimalSerDe : public DataTypeSerDe { int64_t end, const cctz::time_zone& ctz) const override; Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 6d589637f8a309..8a93a5e2b624e1 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -19,6 +19,8 @@ #include +#include +#include #include #include #include @@ -33,6 +35,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/packed_int128.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -51,6 +54,23 @@ namespace doris { namespace { +float parquet_half_to_float(uint16_t half) { + const uint32_t sign = (half & 0x8000U) << 16; + const uint32_t exponent = (half & 0x7C00U) >> 10; + const uint32_t mantissa = half & 0x03FFU; + if (exponent == 0) { + if (mantissa == 0) { + return std::bit_cast(sign); + } + const float value = std::ldexp(static_cast(mantissa), -24); + return sign == 0 ? value : -value; + } + if (exponent == 0x1FU) { + return std::bit_cast(sign | 0x7F800000U | (mantissa << 13)); + } + return std::bit_cast(sign | ((exponent + 112U) << 23) | (mantissa << 13)); +} + template const NativeType* decoded_values_as(const DecodedColumnView& view) { return reinterpret_cast(view.values); @@ -179,6 +199,148 @@ Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& vie } } +template +Status append_parquet_number(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load(values + row * sizeof(SourceType)); + if (!decoded_number_value_fits(value)) { + data.resize(old_size); + return Status::DataQualityError("Parquet value is out of range at row {}", row); + } + data[old_size + row] = static_cast(value); + } + return Status::OK(); +} + +template +Status append_parquet_logical_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values) { + const size_t old_size = data.size(); + data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); + const auto logical_value = static_cast(physical_value); + if (!decoded_number_value_fits(logical_value)) { + data.resize(old_size); + return Status::DataQualityError("Parquet logical integer is out of range at row {}", + row); + } + data[old_size + row] = static_cast(logical_value); + } + return Status::OK(); +} + +template +Status append_parquet_integers(PaddedPODArray& data, const uint8_t* values, + size_t num_values, const ParquetDecodeContext& context) { + if (context.logical_integer_bit_width <= 0) { + return append_parquet_number(data, values, num_values, context); + } + if (context.logical_integer_is_signed) { + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers(data, values, + num_values); + case 16: + return append_parquet_logical_integers(data, values, + num_values); + case 32: + return append_parquet_logical_integers(data, values, + num_values); + case 64: + return append_parquet_logical_integers(data, values, + num_values); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } + } + switch (context.logical_integer_bit_width) { + case 8: + return append_parquet_logical_integers(data, values, + num_values); + case 16: + return append_parquet_logical_integers(data, values, + num_values); + case 32: + return append_parquet_logical_integers(data, values, + num_values); + case 64: + return append_parquet_logical_integers(data, values, + num_values); + default: + return Status::NotSupported("Unsupported Parquet integer bit width {}", + context.logical_integer_bit_width); + } +} + +template +class NumberParquetConsumer final : public ParquetFixedValueConsumer { +public: + using DorisCppType = typename PrimitiveTypeTraits::CppType; + using ColumnType = typename PrimitiveTypeTraits::ColumnType; + + NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context) + : _data(assert_cast(column).get_data()), _context(context) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (_context.logical_float16) { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY); + DORIS_CHECK_EQ(value_width, sizeof(uint16_t)); + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const float value = parquet_half_to_float( + unaligned_load(values + row * sizeof(uint16_t))); + if (!decoded_number_value_fits(value)) { + _data.resize(old_size); + return Status::DataQualityError( + "Parquet FLOAT16 value is out of range at row {}", row); + } + _data[old_size + row] = static_cast(value); + } + return Status::OK(); + } + switch (_context.physical_type) { + case ParquetPhysicalType::BOOLEAN: + DORIS_CHECK_EQ(value_width, sizeof(uint8_t)); + return append_parquet_number(_data, values, num_values, + _context); + case ParquetPhysicalType::INT32: + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + return append_parquet_integers(_data, values, num_values, + _context); + case ParquetPhysicalType::INT64: + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + return append_parquet_integers(_data, values, num_values, + _context); + case ParquetPhysicalType::FLOAT: + DORIS_CHECK_EQ(value_width, sizeof(float)); + return append_parquet_number(_data, values, num_values, _context); + case ParquetPhysicalType::DOUBLE: + DORIS_CHECK_EQ(value_width, sizeof(double)); + return append_parquet_number(_data, values, num_values, _context); + default: + return Status::NotSupported("Unsupported Parquet physical type {} for numeric SerDe", + static_cast(_context.physical_type)); + } + } + +private: + PaddedPODArray& _data; + const ParquetDecodeContext& _context; +}; + +class RejectParquetBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as a number"); + } +}; + } // namespace // Basic structure of the type map. template @@ -330,6 +492,50 @@ Status DataTypeNumberSerDe::read_column_from_decoded_values( get_name(), static_cast(view.value_kind))); } +template +Status DataTypeNumberSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_parquet_dictionary(column, source, context); + } else { + NumberParquetConsumer consumer(column, context); + RejectParquetBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); + } +} + +template +Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if constexpr (!(T == TYPE_BOOLEAN || T == TYPE_TINYINT || T == TYPE_SMALLINT || T == TYPE_INT || + T == TYPE_BIGINT || T == TYPE_LARGEINT || T == TYPE_FLOAT || + T == TYPE_DOUBLE)) { + return DataTypeSerDe::read_column_from_parquet(column, source, context, num_values, state); + } else { + NumberParquetConsumer consumer(column, context); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); + } +} + template Status DataTypeNumberSerDe::deserialize_one_cell_from_json(IColumn& column, Slice& slice, const FormatOptions& options) const { diff --git a/be/src/core/data_type_serde/data_type_number_serde.h b/be/src/core/data_type_serde/data_type_number_serde.h index 4ced14ed858f94..49196d3956d01e 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.h +++ b/be/src/core/data_type_serde/data_type_number_serde.h @@ -119,6 +119,11 @@ class DataTypeNumberSerDe : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, diff --git a/be/src/core/data_type_serde/data_type_serde.cpp b/be/src/core/data_type_serde/data_type_serde.cpp index 6c41b4e9f141db..c84203b6d7f47b 100644 --- a/be/src/core/data_type_serde/data_type_serde.cpp +++ b/be/src/core/data_type_serde/data_type_serde.cpp @@ -52,6 +52,7 @@ #include "core/data_type_serde/data_type_number_serde.h" #include "core/data_type_serde/data_type_string_serde.h" #include "core/data_type_serde/data_type_timestamptz_serde.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/field.h" #include "core/types.h" #include "core/value/timestamptz_value.h" @@ -710,6 +711,18 @@ Status DataTypeSerDe::read_column_from_decoded_values(IColumn& column, get_name())); } +Status DataTypeSerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + return Status::NotSupported("read_column_from_parquet is not supported for {}", get_name()); +} + +Status DataTypeSerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + return Status::NotSupported("read_parquet_dictionary is not supported for {}", get_name()); +} + Status DataTypeSerDe::read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const { return Status::NotSupported("read_column_from_orc is not supported for {}", get_name()); diff --git a/be/src/core/data_type_serde/data_type_serde.h b/be/src/core/data_type_serde/data_type_serde.h index edeb0b0de28781..49bfb73cbd854c 100644 --- a/be/src/core/data_type_serde/data_type_serde.h +++ b/be/src/core/data_type_serde/data_type_serde.h @@ -93,6 +93,9 @@ struct CastParameters; class DataTypeSerDe; using DataTypeSerDeSPtr = std::shared_ptr; using DataTypeSerDeSPtrs = std::vector; +class ParquetDecodeSource; +struct ParquetDecodeContext; +struct ParquetMaterializationState; /// Info that represents a scalar or array field in a decomposed view. /// It allows to recreate field with different number @@ -505,6 +508,18 @@ class DataTypeSerDe { // the Doris-type-specific materialization into IColumn. virtual Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const; + + // Read encoded Parquet values directly into the destination Doris column. ColumnReader owns + // levels/null/filter handling; the source owns only encoding-stream state; the target SerDe + // owns all physical/logical type interpretation and materialization. + virtual Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const; + // Decode one dictionary page into the selected Doris type without consuming data-page + // indices. Dictionary filters use this to keep type interpretation in SerDe instead of + // exposing dictionary bytes or decoder-owned strings to ColumnReader. + virtual Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const; virtual Status read_field_from_decoded_value(const IDataType& data_type, Field* field, const DecodedColumnView& view) const; diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index bd1c6e5b4f19e2..0d2d6c4034234e 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -17,14 +17,18 @@ #include "core/data_type_serde/data_type_string_serde.h" +#include #include #include +#include #include "common/config.h" #include "core/column/column_string.h" +#include "core/column/column_vector.h" #include "core/data_type/define_primitive_type.h" #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "util/jsonb_document_cast.h" #include "util/jsonb_utils.h" #include "util/jsonb_writer.h" @@ -57,6 +61,38 @@ Status read_string_decoded_values(IColumn& column, const DecodedColumnView& view return Status::OK(); } +template +class StringParquetConsumer final : public ParquetFixedValueConsumer, + public ParquetBinaryValueConsumer { +public: + explicit StringParquetConsumer(IColumn& column) : _column(assert_cast(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + static constexpr size_t BATCH_SIZE = 256; + std::array refs; + size_t offset = 0; + while (offset < num_values) { + const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); + for (size_t row = 0; row < batch_size; ++row) { + refs[row] = StringRef( + reinterpret_cast(values + (offset + row) * value_width), + value_width); + } + _column.insert_many_strings(refs.data(), batch_size); + offset += batch_size; + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + _column.insert_many_strings(values, num_values); + return Status::OK(); + } + +private: + ColumnType& _column; +}; + } // namespace namespace { @@ -496,6 +532,61 @@ Status DataTypeStringSerDeBase::read_column_from_decoded_values( return read_string_decoded_values(column, view); } +template +Status DataTypeStringSerDeBase::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + StringParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +template +Status DataTypeStringSerDeBase::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.dictionary_index_only) { + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return Status::IOError("Dictionary filter requested for a non-dictionary page"); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + auto& indices = assert_cast(column).get_data(); + const size_t old_size = indices.size(); + indices.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(state.dictionary_indices[row] > + static_cast(std::numeric_limits::max()))) { + indices.resize(old_size); + return Status::Corruption("Parquet dictionary index {} exceeds INT32", + state.dictionary_indices[row]); + } + indices[old_size + row] = static_cast(state.dictionary_indices[row]); + } + return Status::OK(); + } + StringParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type {} for string SerDe", + static_cast(context.physical_type)); + } + + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} + template Status DataTypeStringSerDeBase::write_column_to_orc( const std::string& timezone, const IColumn& column, const NullMap* null_map, diff --git a/be/src/core/data_type_serde/data_type_string_serde.h b/be/src/core/data_type_serde/data_type_string_serde.h index ad9a05dc996358..c81f6d37aa4ffc 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.h +++ b/be/src/core/data_type_serde/data_type_string_serde.h @@ -205,6 +205,11 @@ class DataTypeStringSerDeBase : public DataTypeSerDe { Status read_column_from_decoded_values(IColumn& column, const DecodedColumnView& view) const override; + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; Status read_column_from_orc(IColumn& column, const OrcDecodedColumnView& view) const override; Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& result, diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index c40e671793c848..8c22f8a1ae336b 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -21,9 +21,11 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" #include "core/data_type_serde/decoded_column_view.h" +#include "core/data_type_serde/parquet_decode_source.h" #include "core/value/time_value.h" #include "exprs/function/cast/cast_base.h" #include "exprs/function/cast/cast_to_time_impl.hpp" +#include "util/unaligned.h" namespace doris { namespace { @@ -51,6 +53,56 @@ TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64 abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); } +class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { +public: + TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context) + : _data(assert_cast(column).get_data()), _context(context) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + int64_t micros; + if (_context.physical_type == ParquetPhysicalType::INT32) { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + micros = static_cast( + unaligned_load(values + row * sizeof(int32_t))) * + 1000; + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + micros = unaligned_load(values + row * sizeof(int64_t)); + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + micros /= 1000; + } + } + const bool negative = micros < 0; + const uint64_t abs_micros = negative ? uint64_t(-(micros + 1)) + 1 : uint64_t(micros); + _data[old_size + row] = + TimeValue::make_time(abs_micros / TimeValue::ONE_HOUR_MICROSECONDS, + (abs_micros % TimeValue::ONE_HOUR_MICROSECONDS) / + TimeValue::ONE_MINUTE_MICROSECONDS, + (abs_micros % TimeValue::ONE_MINUTE_MICROSECONDS) / + TimeValue::ONE_SECOND_MICROSECONDS, + abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); + } + return Status::OK(); + } + +private: + ColumnTimeV2::Container& _data; + const ParquetDecodeContext& _context; +}; + +class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMEV2"); + } +}; + } // namespace Status DataTypeTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, @@ -193,6 +245,39 @@ Status DataTypeTimeV2SerDe::read_column_from_decoded_values(IColumn& column, return Status::OK(); } +Status DataTypeTimeV2SerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + TimeV2ParquetConsumer consumer(column, context); + RejectTimeV2BinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + if ((context.physical_type != ParquetPhysicalType::INT32 && + context.physical_type != ParquetPhysicalType::INT64) || + context.logical_type != ParquetLogicalType::TIME) { + return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); + } + TimeV2ParquetConsumer consumer(column, context); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} + template Status DataTypeTimeV2SerDe::from_int_batch(const typename IntDataType::ColumnType& int_col, ColumnNullable& target_col) const { diff --git a/be/src/core/data_type_serde/data_type_time_serde.h b/be/src/core/data_type_serde/data_type_time_serde.h index e3fccf379c913a..20fa14a148854a 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.h +++ b/be/src/core/data_type_serde/data_type_time_serde.h @@ -69,6 +69,11 @@ class DataTypeTimeV2SerDe : public DataTypeNumberSerDe(column).get_data()), _context(context) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + for (size_t row = 0; row < num_values; ++row) { + int64_t timestamp_micros; + if (_context.physical_type == ParquetPhysicalType::INT96) { + DORIS_CHECK_EQ(value_width, sizeof(DecodedInt96Timestamp)); + timestamp_micros = unaligned_load( + values + row * sizeof(DecodedInt96Timestamp)) + .to_timestamp_micros(); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + timestamp_micros = unaligned_load(values + row * sizeof(int64_t)); + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + timestamp_micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + timestamp_micros /= 1000; + } + } + append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); + } + return Status::OK(); + } + +private: + ColumnTimeStampTz::Container& _data; + const ParquetDecodeContext& _context; +}; + +class RejectTimestampTzBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + return Status::NotSupported("Binary Parquet values cannot be materialized as TIMESTAMPTZ"); + } +}; + } // namespace // The implementation of these functions mainly refers to data_type_datetimev2_serde.cpp @@ -331,6 +373,37 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( return Status::OK(); } +Status DataTypeTimeStampTzSerDe::read_parquet_dictionary( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context) const { + TimestampTzParquetConsumer consumer(column, context); + RejectTimestampTzBinaryConsumer binary_consumer; + return source.decode_dictionary(consumer, binary_consumer); +} + +Status DataTypeTimeStampTzSerDe::read_column_from_parquet( + IColumn& column, ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) const { + if (context.physical_type != ParquetPhysicalType::INT64 && + context.physical_type != ParquetPhysicalType::INT96) { + return Status::NotSupported("TIMESTAMPTZ expects Parquet INT64 or INT96"); + } + TimestampTzParquetConsumer consumer(column, context); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} + std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { return CastToString::from_timestamptz(field.get(), 6); } diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.h b/be/src/core/data_type_serde/data_type_timestamptz_serde.h index 23d57f57fc8dac..6777459909a090 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.h @@ -75,6 +75,11 @@ class DataTypeTimeStampTzSerDe : public DataTypeNumberSerDe(column)) {} + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(reinterpret_cast(values + row * value_width), + value_width); + } + return Status::OK(); + } + + Status consume(const StringRef* values, size_t num_values) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(values[row].data, values[row].size); + } + return Status::OK(); + } + +private: + ColumnVarbinary& _column; +}; + +} // namespace + +Status DataTypeVarbinarySerDe::read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const { + VarbinaryParquetConsumer consumer(column); + return source.decode_dictionary(consumer, consumer); +} + +Status DataTypeVarbinarySerDe::read_column_from_parquet(IColumn& column, + ParquetDecodeSource& source, + const ParquetDecodeContext& context, + size_t num_values, + ParquetMaterializationState& state) const { + VarbinaryParquetConsumer consumer(column); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { + return source.decode_binary_values(num_values, consumer); + } + if (context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY) { + return source.decode_fixed_values(num_values, consumer); + } + return Status::NotSupported("Unsupported Parquet physical type for VARBINARY"); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + return Status::OK(); +} void DataTypeVarbinarySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena& arena, int32_t col_id, int64_t row_num, diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.h b/be/src/core/data_type_serde/data_type_varbinary_serde.h index 41dfa3fce002f8..6d23159c8ea5fe 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.h +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.h @@ -81,6 +81,12 @@ class DataTypeVarbinarySerDe : public DataTypeSerDe { "read_column_from_arrow with type " + column.get_name()); } + Status read_column_from_parquet(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) const override; + Status read_parquet_dictionary(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context) const override; + Status write_column_to_mysql_binary(const IColumn& column, MysqlRowBinaryBuffer& row_buffer, int64_t row_idx, bool col_const, const FormatOptions& options) const override; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h new file mode 100644 index 00000000000000..bd7d93d96ee8e1 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "core/string_ref.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris { + +// These enums deliberately do not expose parquet thrift classes to the core type system. The +// format reader translates the thrift metadata once when it creates a column reader. +enum class ParquetPhysicalType { + BOOLEAN, + INT32, + INT64, + INT96, + FLOAT, + DOUBLE, + BYTE_ARRAY, + FIXED_LEN_BYTE_ARRAY, +}; + +enum class ParquetValueEncoding { + PLAIN, + DICTIONARY, + RLE, + BIT_PACKED, + DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, + DELTA_BYTE_ARRAY, + BYTE_STREAM_SPLIT, +}; + +enum class ParquetTimeUnit { + UNKNOWN, + MILLIS, + MICROS, + NANOS, +}; + +enum class ParquetLogicalType { + NONE, + STRING, + DECIMAL, + DATE, + TIME, + TIMESTAMP, + INTEGER, + UUID, + FLOAT16, +}; + +// Immutable metadata required to turn one Parquet physical value into the selected Doris type. +// Encoding describes how the value source is read; logical annotations describe its meaning. +struct ParquetDecodeContext { + ParquetPhysicalType physical_type = ParquetPhysicalType::INT32; + ParquetValueEncoding encoding = ParquetValueEncoding::PLAIN; + ParquetLogicalType logical_type = ParquetLogicalType::NONE; + ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; + + int32_t type_length = -1; + int32_t decimal_precision = -1; + int32_t decimal_scale = -1; + int32_t logical_integer_bit_width = -1; + bool logical_integer_is_signed = true; + bool timestamp_is_adjusted_to_utc = false; + bool logical_float16 = false; + bool logical_uuid = false; + bool dictionary_index_only = false; + + const cctz::time_zone* timezone = nullptr; +}; + +// A decoder may produce multiple contiguous spans for one request (for example delta encodings). +// Consumers are invoked per span, never per value, keeping virtual dispatch out of the row loop. +class ParquetFixedValueConsumer { +public: + virtual ~ParquetFixedValueConsumer() = default; + virtual Status consume(const uint8_t* values, size_t num_values, size_t value_width) = 0; +}; + +class ParquetBinaryValueConsumer { +public: + virtual ~ParquetBinaryValueConsumer() = default; + virtual Status consume(const StringRef* values, size_t num_values) = 0; +}; + +// Encoding decoders implement this interface. They own encoded-stream cursors and dictionary +// storage, but they never know the destination Doris column type. DataTypeSerDe owns the consumer +// and therefore the physical/logical-to-Doris conversion. +class ParquetDecodeSource { +public: + virtual ~ParquetDecodeSource() = default; + + virtual Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) = 0; + virtual Status decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) = 0; + virtual Status skip_values(size_t num_values) = 0; + + virtual bool has_dictionary() const { return false; } + virtual uint64_t dictionary_generation() const { return 0; } + virtual size_t dictionary_size() const { return 0; } + virtual Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return Status::NotSupported("Parquet dictionary is not supported by this decoder"); + } + virtual Status decode_dictionary_indices(size_t num_values, std::vector* indices) { + return Status::NotSupported("Parquet dictionary indices are not supported by this decoder"); + } +}; + +// Dictionary values are materialized once into the selected Doris type. The state belongs to a +// column reader rather than DataTypeSerDe because a SerDe instance can be shared by many files. +struct ParquetMaterializationState { + MutableColumnPtr typed_dictionary; + std::vector dictionary_indices; + uint64_t dictionary_generation = std::numeric_limits::max(); + + void reset_dictionary() { + typed_dictionary.reset(); + dictionary_indices.clear(); + dictionary_generation = std::numeric_limits::max(); + } +}; + +} // namespace doris diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index c669de184076c6..3d7fddeec5043e 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -111,10 +111,10 @@ instructions as well; this file adds format-v2-specific review expectations. ### Parquet Native Decode Kernel - Keep new production integration under `be/src/format_v2/parquet/`. Doris v1 is the behavior and - performance baseline. A v2 adapter may reuse the unchanged native kernel while its semantics are - identical; do not modify `be/src/format/parquet/` for a v2 decoder change. When v2 requires a - decoder change, reimplement that decoder under the v2 tree and keep v1 unchanged so differential - correctness and performance results remain meaningful. + performance baseline, but v2 owns an independent page/encoding reader and must not call the v1 + `ParquetColumnReader`. Do not modify `be/src/format/parquet/` for a v2 decoder change. Reimplement + the required behavior under the v2 tree and keep v1 unchanged so differential correctness and + performance results remain meaningful. - Keep the native decode boundary independent of both Arrow descriptors/builders and table-schema objects. A Column Chunk schema contract should contain only immutable physical type, fixed width, and Dremel-level thresholds. Review constructor arguments and stored references for ownership and @@ -128,24 +128,29 @@ instructions as well; this file adds format-v2-specific review expectations. selected and filtered non-null runs both consume payload. Every page transition, skip, error, and end-of-batch path must leave all three cursors aligned for the next call. - A flat scalar fast path may run only when `max_repetition_level == 0`. Repeated leaves require a - shared definition/repetition-level plan that identifies parent-row boundaries, empty and null - collections, null ancestors, and rows spanning pages. Sibling readers in a STRUCT, ARRAY, or MAP - must consume the same parent-row plan rather than independently inferring offsets or null maps. + definition/repetition-level plan that identifies parent-row boundaries, empty and null + collections, null ancestors, and rows spanning pages. Choose one physical leaf as the parent + shape owner; sibling leaf streams advance over the same parent-row range and validate their + payload counts instead of independently redefining the parent shape. - The old Arrow value-reader hierarchy (`ParquetLeafBatch`, scalar/list/map/struct readers, and the nested load/build/consume protocol) has been removed. Do not reintroduce an intermediate decoded batch or a stateful load-before-build phase. Ordinary predicate/output scans construct `NativeColumnReader`, consume compressed page data through the native decoder, and append directly into the final Doris column. -- `CountColumnReader` is the sole Arrow data-page exception. It is an isolated shape-only adapter - for the existing `COUNT(nullable_col)` pushdown because Arrow exposes no independent public level - decoder. It selects one representative leaf (the key for MAP), copies only definition/repetition - levels, immediately releases binary builder chunks, and exposes no value API. It must not be +- Keep logical schema changes distinct from physical decoding. Identical types, integer changes, + FLOAT-to-DOUBLE widening, decimal precision/scale changes, and string-family changes should + materialize through the target SerDe directly. A less common logical cast may use the generic + `ColumnTypeConverter` with a reusable source Doris column, but must not revive + `PhysicalToLogicalConverter` or expose a decoder-owned value batch. +- `CountColumnReader` uses the v2 native `LevelReader`; it selects one representative leaf (the key + for MAP) and advances only definition/repetition levels. It exposes no value API and must not be reused as a scan reader or expanded into a fallback path. -- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one level - traversal and share the result. For example, `[[1, 2], NULL, []]` must yield entry counts - `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning the same levels per child. MAP key - levels own entry existence; value levels validate against that plan. STRUCT siblings validate - alignment but do not independently reconstruct the parent shape. +- Build ARRAY/MAP/STRUCT parent boundaries, offsets, nulls, and child payload spans in one traversal + of the owning leaf's levels. For example, `[[1, 2], NULL, []]` must yield entry counts + `[2, 0, 0]` and parent nulls `[0, 1, 0]` without rescanning that leaf. MAP key levels own entry + existence; value levels validate against that shape. STRUCT siblings validate parent-row + alignment. If every projected STRUCT child is missing, consume only a retained physical leaf's + levels and never materialize its payload into a temporary Doris column. - Do not size level or selection scratch from a 16-bit batch-row assumption. A repeated parent row can contain more level entries than the requested parent-row batch. Split large runs without changing alternation or row-boundary semantics, and check overflow before narrowing counts. @@ -165,15 +170,15 @@ instructions as well; this file adds format-v2-specific review expectations. indexes retain candidates; structurally inconsistent indexes or out-of-range IDs return an explicit corruption error. A mixed dictionary/plain Column Chunk must leave dictionary-ID filtering before any cursor is consumed. -- Materialize directly into Doris columns when the physical and target layouts allow it. Decimal - and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, precision, - and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. A direct - path may not bypass the conversion rules used by the general conversion path. +- Decoder owns encoded-stream parsing and cursor movement only. `DataTypeSerDe` owns Parquet + physical/logical interpretation and writes directly into Doris columns. Dictionary pages are + materialized once through the same SerDe and data pages expose only validated dictionary indices. + Decimal and FIXED_LEN_BYTE_ARRAY paths must validate byte width, endianness, sign extension, + precision, and scale. Date/time and INT96 conversion must preserve timezone and overflow semantics. - Do not add an Arrow runtime fallback. Once an ordinary v2 scan selects its native Parquet reader, unsupported physical types, encodings, page layouts, or malformed inputs return an explicit - status. Arrow may be used by metadata planning, the isolated COUNT shape adapter, and as a test - oracle; no Arrow array, builder, RecordReader, or metadata lifetime belongs in ordinary - value materialization. + status. Arrow may be used by metadata planning and as a test oracle; no Arrow array, builder, + RecordReader, or metadata lifetime belongs in data-page value or level materialization. - Reuse decoder, SerDe, null-map, selection-range, binary-value, level, and builder scratch across batches. String-like decoders should gather selected `StringRef` values and append once per batch, rather than allocate or grow the destination once per run. Scratch capacity may grow to a bounded diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 9f37ea2ca49882..676c6631f084fc 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -73,10 +73,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); reader_select_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "ReaderSelectRows", TUnit::UNIT, parquet_profile, 1); - arrow_read_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowReadRecordsTime", parquet_profile, 1); - arrow_skip_records_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowSkipRecordsTime", parquet_profile, 1); + level_only_read_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlyReadTime", parquet_profile, 1); + level_only_skip_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, @@ -211,8 +211,8 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .reader_read_rows = reader_read_rows, .reader_skip_rows = reader_skip_rows, .reader_select_rows = reader_select_rows, - .arrow_read_records_time = arrow_read_records_time, - .arrow_skip_records_time = arrow_skip_records_time, + .level_only_read_time = level_only_read_time, + .level_only_skip_time = level_only_skip_time, .materialization_time = materialization_time, .decompress_time = decompress_time, .decompress_count = decompress_cnt, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index f0cdfd4d00ab66..ef2cb452667278 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -31,13 +31,13 @@ struct ParquetPageSkipProfile { // ============================================================================ // ============================================================================ struct ParquetColumnReaderProfile { - RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() - RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() - RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() - // COUNT(nullable_col) shape-only compatibility path; ordinary scans keep both counters zero. - RuntimeProfile::Counter* arrow_read_records_time = nullptr; - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; - RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* reader_read_rows = nullptr; // rows read by read() + RuntimeProfile::Counter* reader_skip_rows = nullptr; // rows skipped by skip() + RuntimeProfile::Counter* reader_select_rows = nullptr; // rows selected by select() + // COUNT(nullable_col) shape-only path; ordinary scans keep both counters zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; + RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) // Native page/encoding reader internals. These counters intentionally mirror v1 so a v1/v2 // profile comparison attributes page IO, decompression, levels, value decode and conversion to // the same stages. @@ -136,9 +136,9 @@ struct ParquetProfile { RuntimeProfile::Counter* reader_read_rows = nullptr; RuntimeProfile::Counter* reader_skip_rows = nullptr; RuntimeProfile::Counter* reader_select_rows = nullptr; - // COUNT(nullable_col) shape-only compatibility path; ordinary scans keep these zero. - RuntimeProfile::Counter* arrow_read_records_time = nullptr; - RuntimeProfile::Counter* arrow_skip_records_time = nullptr; + // COUNT(nullable_col) shape-only path; ordinary scans keep these zero. + RuntimeProfile::Counter* level_only_read_time = nullptr; + RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; RuntimeProfile::Counter* native_read_calls = nullptr; RuntimeProfile::Counter* native_page_fragments = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 6cc4f69277fb45..c3453e8749e548 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -213,8 +213,7 @@ const ParquetColumnSchema& projected_root_schema( } int64_t count_loaded_non_null_values(const ParquetColumnSchema& root_schema, - const CountColumnReader& shape_reader, - int64_t expected_rows) { + const CountColumnReader& shape_reader, int64_t expected_rows) { const auto& def_levels = shape_reader.definition_levels(); const auto& rep_levels = shape_reader.repetition_levels(); const int64_t levels_written = shape_reader.levels_written(); @@ -711,26 +710,12 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } result->count = 0; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - std::shared_ptr<::parquet::RowGroupReader> row_group; - try { - row_group = _state->file_context.file_reader->RowGroup(row_group_plan.row_group_id); - } catch (const ::parquet::ParquetException& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } catch (const std::exception& e) { - if (_should_stop()) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet row group {}: {}", - row_group_plan.row_group_id, e.what()); - } - std::unique_ptr shape_reader; RETURN_IF_ERROR(CountColumnReader::create( - row_group, root_schema, &count_projection, + _state->file_context.native_data_file(), _state->file_context.native_metadata, + row_group_plan.row_group_id, root_schema, &count_projection, + _state->file_context.native_io_ctx, + _state->file_context.native_page_cache_enabled, _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 8880bf4ccaec73..8790973566cdf0 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -823,17 +823,15 @@ Status ParquetScanScheduler::open_next_row_group( continue; } if (local_id == format::ROW_POSITION_COLUMN_ID) { - _current_non_predicate_columns[local_id] = - std::make_unique( - _current_row_group_first_row, _scan_profile.column_reader_profile); + _current_non_predicate_columns[local_id] = std::make_unique( + _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { DORIS_CHECK(_global_rowid_context.has_value()); - _current_non_predicate_columns[local_id] = - std::make_unique( - *_global_rowid_context, _current_row_group_first_row, - _scan_profile.column_reader_profile); + _current_non_predicate_columns[local_id] = std::make_unique( + *_global_rowid_context, _current_row_group_first_row, + _scan_profile.column_reader_profile); continue; } DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 6ea0e62960fb5e..42312ce8ed8de4 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -55,10 +55,10 @@ class ParquetColumnReader { int64_t batch_rows, MutableColumnPtr& column); virtual Status select_with_dictionary_filter(const SelectionVector& selection, - uint16_t selected_rows, int64_t batch_rows, - const IColumn::Filter& dictionary_filter, - MutableColumnPtr& column, - IColumn::Filter* row_filter, bool* used_filter); + uint16_t selected_rows, int64_t batch_rows, + const IColumn::Filter& dictionary_filter, + MutableColumnPtr& column, + IColumn::Filter* row_filter, bool* used_filter); protected: ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp index 30fd395359574e..1706039f03bc7f 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -5,7 +5,9 @@ // to you under the Apache License, Version 2.0 (the // "License"); you may not use this file except in compliance // with the License. You may obtain a copy of the License at +// // http://www.apache.org/licenses/LICENSE-2.0 +// // Unless required by applicable law or agreed to in writing, // software distributed under the License is distributed on an // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -15,19 +17,15 @@ #include "format_v2/parquet/reader/count_column_reader.h" -#include -#include -#include -#include -#include -#include - #include -#include #include #include +#include "common/config.h" +#include "format/parquet/schema_desc.h" +#include "format/parquet/vparquet_file_metadata.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/reader/native/level_reader.h" #include "runtime/runtime_profile.h" namespace doris::format::parquet { @@ -49,10 +47,11 @@ Status find_count_leaf(const ParquetColumnSchema& schema, if (!format::is_partial_projection(projection)) { return find_count_leaf(*schema.children.front(), nullptr, leaf); } + DORIS_CHECK(!projection->children.empty()); const auto child_id = projection->children.front().local_id(); - const auto child = std::ranges::find_if( - schema.children, - [child_id](const auto& candidate) { return candidate->local_id == child_id; }); + const auto child = std::ranges::find_if(schema.children, [child_id](const auto& candidate) { + return candidate->local_id == child_id; + }); if (child == schema.children.end()) { return Status::InvalidArgument( "Parquet COUNT projection for column {} contains invalid child", schema.name); @@ -62,67 +61,84 @@ Status find_count_leaf(const ParquetColumnSchema& schema, case ParquetColumnSchemaKind::LIST: { DORIS_CHECK(schema.children.size() == 1); const auto& element = *schema.children.front(); - return find_count_leaf(element, - format::find_child_projection(projection, element.local_id), leaf); + return find_count_leaf(element, format::find_child_projection(projection, element.local_id), + leaf); } case ParquetColumnSchemaKind::MAP: - // The key stream defines entry existence and top-level MAP shape. Never select the value - // leaf: a COUNT(map_col) must not retain huge value strings merely to inspect nullability. + // The key leaf owns entry existence and top-level MAP shape. Choosing it also avoids + // reading a potentially huge value BYTE_ARRAY for COUNT(map_col). DORIS_CHECK(!schema.children.empty()); return find_count_leaf(*schema.children.front(), nullptr, leaf); } return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); } -bool is_binary_physical_type(const ::parquet::ColumnDescriptor& descriptor) { - return descriptor.physical_type() == ::parquet::Type::BYTE_ARRAY || - descriptor.physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY; +FieldSchema* find_physical_leaf(FieldSchema* field, int physical_column_index) { + DORIS_CHECK(field != nullptr); + if (field->children.empty()) { + return field->physical_column_index == physical_column_index ? field : nullptr; + } + for (auto& child : field->children) { + if (auto* result = find_physical_leaf(&child, physical_column_index); result != nullptr) { + return result; + } + } + return nullptr; } } // namespace -CountColumnReader::CountColumnReader( - const ParquetColumnSchema& leaf_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, - ParquetColumnReaderProfile profile) - : _leaf_schema(leaf_schema), - _record_reader(std::move(record_reader)), - _profile(profile), - _name(leaf_schema.name) {} - -Status CountColumnReader::create(std::shared_ptr<::parquet::RowGroupReader> row_group, - const ParquetColumnSchema& root_schema, - const format::LocalColumnIndex* projection, - ParquetColumnReaderProfile profile, +CountColumnReader::CountColumnReader(std::string name, + std::unique_ptr level_reader, + ParquetColumnReaderProfile profile) + : _level_reader(std::move(level_reader)), _profile(profile), _name(std::move(name)) {} + +CountColumnReader::~CountColumnReader() { + sync_profile(); +} + +Status CountColumnReader::create(io::FileReaderSPtr file, const FileMetaData* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, ParquetColumnReaderProfile profile, std::unique_ptr* reader) { - DORIS_CHECK(row_group != nullptr); + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); DORIS_CHECK(reader != nullptr); - const ParquetColumnSchema* leaf = nullptr; - RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf)); - DORIS_CHECK(leaf != nullptr); - DORIS_CHECK(leaf->leaf_column_id >= 0); - DORIS_CHECK(leaf->descriptor != nullptr); - - try { - auto page_reader = row_group->GetColumnPageReader(leaf->leaf_column_id); - DORIS_CHECK(page_reader != nullptr); - const auto level_info = - ::parquet::internal::LevelInfo::ComputeLevelInfo(leaf->descriptor); - auto record_reader = ::parquet::internal::RecordReader::Make( - leaf->descriptor, level_info, ::arrow::default_memory_pool(), - /*read_dictionary=*/false, - /*read_dense_for_nullable=*/false); - DORIS_CHECK(record_reader != nullptr); - record_reader->SetPageReader(std::move(page_reader)); - reader->reset(new CountColumnReader(*leaf, std::move(record_reader), profile)); - return Status::OK(); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to create Parquet COUNT reader for column {}: {}", - leaf->name, e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to create Parquet COUNT reader for column {}: {}", - leaf->name, e.what()); - } + const ParquetColumnSchema* leaf_schema = nullptr; + RETURN_IF_ERROR(find_count_leaf(root_schema, projection, &leaf_schema)); + DORIS_CHECK(leaf_schema != nullptr); + DORIS_CHECK(leaf_schema->leaf_column_id >= 0); + + const auto& thrift_metadata = metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::InvalidArgument("Invalid Parquet COUNT row group {}", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + if (leaf_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet COUNT leaf {} for row group {}", + leaf_schema->leaf_column_id, row_group_id); + } + + DORIS_CHECK(root_schema.local_id >= 0); + auto* root_field = + const_cast(metadata->schema().get_column(root_schema.local_id)); + DORIS_CHECK(root_field != nullptr); + auto* leaf_field = find_physical_leaf(root_field, leaf_schema->leaf_column_id); + if (leaf_field == nullptr) { + return Status::Corruption("Cannot resolve Parquet COUNT physical leaf {} for column {}", + leaf_schema->leaf_column_id, root_schema.name); + } + + const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; + const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; + std::unique_ptr level_reader; + RETURN_IF_ERROR(native::LevelReader::create( + std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, + row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, + enable_page_cache, &level_reader)); + reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); + return Status::OK(); } Status CountColumnReader::skip(int64_t rows) { @@ -130,47 +146,14 @@ Status CountColumnReader::skip(int64_t rows) { if (rows == 0) { return Status::OK(); } - int64_t skipped_rows = 0; - try { - _record_reader->Reset(); - SCOPED_TIMER(_profile.arrow_skip_records_time); - while (skipped_rows < rows) { - const int64_t skipped = _record_reader->SkipRecords(rows - skipped_rows); - if (skipped <= 0) { - return Status::Corruption( - "Parquet COUNT reader skipped {} of {} rows for column {}", skipped_rows, - rows, _name); - } - skipped_rows += skipped; - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to skip Parquet COUNT rows for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to skip Parquet COUNT rows for column {}: {}", _name, - e.what()); + { + SCOPED_TIMER(_profile.level_only_skip_time); + RETURN_IF_ERROR(_level_reader->skip_rows(static_cast(rows))); } if (_profile.reader_skip_rows != nullptr) { COUNTER_UPDATE(_profile.reader_skip_rows, rows); } - return Status::OK(); -} - -Status CountColumnReader::release_binary_builder() { - DORIS_CHECK(_leaf_schema.descriptor != nullptr); - if (!is_binary_physical_type(*_leaf_schema.descriptor)) { - return Status::OK(); - } - auto* binary_reader = - dynamic_cast<::parquet::internal::BinaryRecordReader*>(_record_reader.get()); - if (binary_reader == nullptr) { - return Status::InternalError( - "Parquet COUNT binary reader is unavailable for column {}", _name); - } - // GetBuilderChunks transfers builder ownership. Keep the result local so binary payload pages - // are released before the next batch and never overlap adaptive-batch allocations. - auto discarded_chunks = binary_reader->GetBuilderChunks(); - discarded_chunks.clear(); + sync_profile(); return Status::OK(); } @@ -180,68 +163,60 @@ Status CountColumnReader::read_levels(int64_t rows, int64_t* rows_read) { _definition_levels.clear(); _repetition_levels.clear(); _levels_written = 0; - if (rows == 0) { - *rows_read = 0; - return Status::OK(); - } - - try { - _record_reader->Reset(); - _record_reader->Reserve(rows); - { - SCOPED_TIMER(_profile.arrow_read_records_time); - *rows_read = _record_reader->ReadRecords(rows); - } - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to read Parquet COUNT levels for column {}: {}", _name, - e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to read Parquet COUNT levels for column {}: {}", _name, - e.what()); - } - if (*rows_read < 0 || *rows_read > rows) { - return Status::Corruption("Invalid Parquet COUNT row count {} for column {}", *rows_read, - _name); - } - - _levels_written = _record_reader->levels_position(); - if (_levels_written > _record_reader->levels_written()) { + size_t native_rows_read = 0; + { + SCOPED_TIMER(_profile.level_only_read_time); + RETURN_IF_ERROR(_level_reader->read_rows(static_cast(rows), &_repetition_levels, + &_definition_levels, &native_rows_read)); + } + *rows_read = static_cast(native_rows_read); + if (*rows_read != rows || _definition_levels.size() != _repetition_levels.size()) { return Status::Corruption( - "Invalid Parquet COUNT level position {} of {} for column {}", _levels_written, - _record_reader->levels_written(), _name); - } - const auto* descriptor = _leaf_schema.descriptor; - if (_levels_written == 0 && *rows_read > 0 && descriptor->max_definition_level() == 0 && - descriptor->max_repetition_level() == 0) { - _levels_written = *rows_read; + "Parquet COUNT level reader returned {} rows and {}/{} levels for {}", *rows_read, + _definition_levels.size(), _repetition_levels.size(), _name); } + _levels_written = static_cast(_definition_levels.size()); if (_levels_written < *rows_read) { - return Status::Corruption( - "Parquet COUNT returned {} levels for {} rows in column {}", _levels_written, - *rows_read, _name); - } - - _definition_levels.resize(static_cast(_levels_written)); - if (descriptor->max_definition_level() == 0) { - std::fill(_definition_levels.begin(), _definition_levels.end(), 0); - } else { - const auto* levels = _record_reader->def_levels(); - DORIS_CHECK(levels != nullptr || _levels_written == 0); - std::copy_n(levels, _levels_written, _definition_levels.begin()); - } - _repetition_levels.resize(static_cast(_levels_written)); - if (descriptor->max_repetition_level() == 0) { - std::fill(_repetition_levels.begin(), _repetition_levels.end(), 0); - } else { - const auto* levels = _record_reader->rep_levels(); - DORIS_CHECK(levels != nullptr || _levels_written == 0); - std::copy_n(levels, _levels_written, _repetition_levels.begin()); - } - RETURN_IF_ERROR(release_binary_builder()); + return Status::Corruption("Parquet COUNT returned {} levels for {} rows in column {}", + _levels_written, *rows_read, _name); + } if (_profile.reader_read_rows != nullptr) { COUNTER_UPDATE(_profile.reader_read_rows, *rows_read); } + sync_profile(); return Status::OK(); } +void CountColumnReader::sync_profile() { + if (_level_reader == nullptr) { + return; + } + const auto stats = _level_reader->statistics(); + const auto& reported = _reported_native_stats; +#define UPDATE_NATIVE_PROFILE(counter, field) \ + if (_profile.counter != nullptr) { \ + COUNTER_UPDATE(_profile.counter, stats.field - reported.field); \ + } + UPDATE_NATIVE_PROFILE(decompress_time, decompress_time); + UPDATE_NATIVE_PROFILE(decompress_count, decompress_cnt); + UPDATE_NATIVE_PROFILE(decode_header_time, decode_header_time); + UPDATE_NATIVE_PROFILE(decode_value_time, decode_value_time); + UPDATE_NATIVE_PROFILE(decode_dictionary_time, decode_dict_time); + UPDATE_NATIVE_PROFILE(decode_level_time, decode_level_time); + UPDATE_NATIVE_PROFILE(skip_page_header_count, skip_page_header_num); + UPDATE_NATIVE_PROFILE(parse_page_header_count, parse_page_header_num); + UPDATE_NATIVE_PROFILE(read_page_header_time, read_page_header_time); + UPDATE_NATIVE_PROFILE(page_read_count, page_read_counter); + UPDATE_NATIVE_PROFILE(page_cache_write_count, page_cache_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_write_count, page_cache_compressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_write_count, + page_cache_decompressed_write_counter); + UPDATE_NATIVE_PROFILE(page_cache_hit_count, page_cache_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_miss_count, page_cache_missing_counter); + UPDATE_NATIVE_PROFILE(page_cache_compressed_hit_count, page_cache_compressed_hit_counter); + UPDATE_NATIVE_PROFILE(page_cache_decompressed_hit_count, page_cache_decompressed_hit_counter); +#undef UPDATE_NATIVE_PROFILE + _reported_native_stats = stats; +} + } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h index a02c2988ed0eae..86508e4302d1ea 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.h +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -23,31 +23,28 @@ #include "common/status.h" #include "format_v2/column_data.h" #include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/reader/native/level_reader.h" +#include "io/fs/file_reader_writer_fwd.h" -namespace parquet { -class RowGroupReader; -namespace internal { -class RecordReader; -} // namespace internal -} // namespace parquet +namespace doris { +class FileMetaData; +namespace io { +struct IOContext; +} +} // namespace doris namespace doris::format::parquet { struct ParquetColumnSchema; - -// Isolated compatibility reader for the existing COUNT(nullable_col) pushdown. -// -// Ordinary scans never instantiate this class. COUNT needs only Dremel definition/repetition -// levels, so this reader exposes exactly one shape operation and no value-materialization API. -// Arrow currently has no public levels-only page decoder; ReadRecords therefore advances its -// private RecordReader, after which binary builder chunks are immediately released and only the -// copied level vectors survive. Keeping this exception isolated prevents Arrow arrays/builders, -// ParquetLeafBatch, and decoded-value views from leaking back into the scan reader contract. +// Shape-only COUNT(nullable_col) reader. It uses v2's native LevelReader, so BYTE_ARRAY payloads +// are skipped in the encoding stream and never copied into Arrow builders or Doris strings. class CountColumnReader { public: - static Status create(std::shared_ptr<::parquet::RowGroupReader> row_group, + ~CountColumnReader(); + + static Status create(io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, const ParquetColumnSchema& root_schema, - const format::LocalColumnIndex* projection, - ParquetColumnReaderProfile profile, + const format::LocalColumnIndex* projection, io::IOContext* io_ctx, + bool enable_page_cache, ParquetColumnReaderProfile profile, std::unique_ptr* reader); Status skip(int64_t rows); @@ -58,18 +55,16 @@ class CountColumnReader { int64_t levels_written() const { return _levels_written; } private: - CountColumnReader(const ParquetColumnSchema& leaf_schema, - std::shared_ptr<::parquet::internal::RecordReader> record_reader, + CountColumnReader(std::string name, std::unique_ptr level_reader, ParquetColumnReaderProfile profile); + void sync_profile(); - Status release_binary_builder(); - - const ParquetColumnSchema& _leaf_schema; - std::shared_ptr<::parquet::internal::RecordReader> _record_reader; + std::unique_ptr _level_reader; ParquetColumnReaderProfile _profile; std::string _name; std::vector _definition_levels; std::vector _repetition_levels; + native::ColumnChunkReaderStatistics _reported_native_stats; int64_t _levels_written = 0; }; diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp new file mode 100644 index 00000000000000..a585fe8d4234c2 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" + +#include + +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "format/parquet/parquet_common.h" +#include "util/bit_util.h" + +namespace doris::format::parquet::native { +Status BoolPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + std::array values; + size_t decoded = 0; + while (decoded < num_values) { + const size_t batch_size = std::min(values.size(), num_values - decoded); + for (size_t row = 0; row < batch_size; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain decoder"); + } + values[row] = static_cast(value); + } + RETURN_IF_ERROR(consumer.consume(values.data(), batch_size, sizeof(uint8_t))); + decoded += batch_size; + } + return Status::OK(); +} + +Status BoolPlainDecoder::skip_values(size_t num_values) { + int skip_cached = + std::min(num_unpacked_values_ - unpacked_value_idx_, cast_set(num_values)); + unpacked_value_idx_ += skip_cached; + if (skip_cached == num_values) { + return Status::OK(); + } + int num_remaining = cast_set(num_values - skip_cached); + int num_to_skip = BitUtil::RoundDownToPowerOf2(num_remaining, 32); + if (num_to_skip > 0) { + bool_values_.SkipBatch(1, num_to_skip); + } + num_remaining -= num_to_skip; + if (num_remaining > 0) { + DCHECK_LE(num_remaining, UNPACKED_BUFFER_LEN); + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ < num_remaining)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } + unpacked_value_idx_ = num_remaining; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h new file mode 100644 index 00000000000000..c44c4934d8ce30 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris { +class ColumnSelectVector; +} // namespace doris + +namespace doris::format::parquet::native { +/// Decoder bit-packed boolean-encoded values. +/// Implementation from https://github.com/apache/impala/blob/master/be/src/exec/parquet/parquet-bool-decoder.h +//bit-packed-run-len and rle-run-len must be in the range [1, 2^31 - 1]. +// This means that a Parquet implementation can always store the run length in a signed 32-bit integer +class BoolPlainDecoder final : public Decoder { +public: + BoolPlainDecoder() = default; + ~BoolPlainDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + bool_values_.Reset((const uint8_t*)data->data, data->size); + num_unpacked_values_ = 0; + unpacked_value_idx_ = 0; + _offset = 0; + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + +protected: + inline bool _decode_value(bool* value) { + if (LIKELY(unpacked_value_idx_ < num_unpacked_values_)) { + *value = unpacked_values_[unpacked_value_idx_++]; + } else { + num_unpacked_values_ = + bool_values_.UnpackBatch(1, UNPACKED_BUFFER_LEN, &unpacked_values_[0]); + if (UNLIKELY(num_unpacked_values_ == 0)) { + return false; + } + *value = unpacked_values_[0]; + unpacked_value_idx_ = 1; + } + return true; + } + + /// A buffer to store unpacked values. Must be a multiple of 32 size to use the + /// batch-oriented interface of BatchedBitReader. We use uint8_t instead of bool because + /// bit unpacking is only supported for unsigned integers. The values are converted to + /// bool when returned to the user. + static const int UNPACKED_BUFFER_LEN = 128; + uint8_t unpacked_values_[UNPACKED_BUFFER_LEN]; + + /// The number of valid values in 'unpacked_values_'. + int num_unpacked_values_ = 0; + + /// The next value to return from 'unpacked_values_'. + int unpacked_value_idx_ = 0; + + /// Bit packed decoder, used if 'encoding_' is PLAIN. + BatchedBitReader bool_values_; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp new file mode 100644 index 00000000000000..12322e05b033ac --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" + +#include + +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/types.h" +#include "format/parquet/parquet_common.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +Status BoolRLEDecoder::set_data(Slice* slice) { + _data = slice; + _num_bytes = slice->size; + _offset = 0; + if (_num_bytes < 4) { + return Status::IOError("Received invalid length : " + std::to_string(_num_bytes) + + " (corrupt data page?)"); + } + // Load the first 4 bytes in little-endian, which indicates the length + const auto* data = reinterpret_cast(_data->data); + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > static_cast(_num_bytes - 4)) { + return Status::IOError("Received invalid number of bytes : " + std::to_string(num_bytes) + + " (corrupt data page?)"); + } + _num_bytes = num_bytes; + auto decoder_data = data + 4; + _decoder = RleDecoder(decoder_data, num_bytes, 1); + return Status::OK(); +} + +Status BoolRLEDecoder::skip_values(size_t num_values) { + _decoder.Skip(num_values); + return Status::OK(); +} + +Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { + _values.resize(num_values); + if (!_decoder.get_values(_values.data(), num_values)) { + return Status::IOError("Can't read enough booleans in Parquet RLE decoder"); + } + return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h new file mode 100644 index 00000000000000..502e724b359425 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/rle_encoding.h" + +namespace doris { +class ColumnSelectVector; +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +class BoolRLEDecoder final : public Decoder { +public: + BoolRLEDecoder() = default; + ~BoolRLEDecoder() override = default; + + Status set_data(Slice* slice) override; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + +private: + RleDecoder _decoder; + std::vector _values; + size_t _num_bytes; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp new file mode 100644 index 00000000000000..94a9a50bdfbd16 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" + +#include "core/custom_allocator.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { +Status ByteArrayDictDecoder::set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + _dict_items.clear(); + _dict = std::move(dict); + if (_dict == nullptr) { + return Status::Corruption("Wrong dictionary data for byte array type, dict is null."); + } + _dict_items.reserve(num_values); + if (UNLIKELY(length < 0)) { + return Status::Corruption("Wrong data length in dictionary"); + } + const size_t dict_length = cast_set(length); + size_t offset_cursor = 0; + char* dict_item_address = reinterpret_cast(_dict.get()); + for (int i = 0; i < num_values; ++i) { + if (UNLIKELY(offset_cursor > dict_length || + dict_length - offset_cursor < sizeof(uint32_t))) { + return Status::Corruption("Wrong data length in dictionary"); + } + uint32_t l = decode_fixed32_le(_dict.get() + offset_cursor); + offset_cursor += sizeof(uint32_t); + if (UNLIKELY(l > dict_length - offset_cursor)) { + return Status::Corruption("Wrong data length in dictionary"); + } + _dict_items.emplace_back(dict_item_address + offset_cursor, l); + offset_cursor += l; + } + if (offset_cursor != dict_length) { + return Status::Corruption("Wrong dictionary data for byte array type"); + } + ++_dictionary_generation; + return Status::OK(); +} + +Status ByteArrayDictDecoder::decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) { + return binary_consumer.consume(_dict_items.data(), _dict_items.size()); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h new file mode 100644 index 00000000000000..790d9f2e4b65ba --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.h @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/string_ref.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteArrayDictDecoder final : public BaseDictDecoder { +public: + ByteArrayDictDecoder() = default; + ~ByteArrayDictDecoder() override = default; + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override; + + size_t dictionary_size() const override { return _dict_items.size(); } + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override; + +protected: + // StringRef entries point into BaseDictDecoder::_dict. The dictionary buffer lives for the + // entire column chunk, so retaining a second copy of every byte is unnecessary. + DorisVector _dict_items; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp new file mode 100644 index 00000000000000..df756abe2c904a --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" + +#include +#include + +#include "core/column/column.h" +#include "core/data_type/data_type_nullable.h" +#include "core/string_ref.h" + +namespace doris::format::parquet::native { +namespace { +Status read_length(const Slice* data, uint32_t* offset, uint32_t* length) { + if (UNLIKELY(*offset > data->size || data->size - *offset < sizeof(uint32_t))) { + return Status::IOError("Can't read byte array length from plain decoder"); + } + *length = decode_fixed32_le(reinterpret_cast(data->data) + *offset); + *offset += sizeof(uint32_t); + return Status::OK(); +} +} // namespace + +Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, + ParquetBinaryValueConsumer& consumer) { + _binary_values.clear(); + _binary_values.reserve(num_values); + for (size_t row = 0; row < num_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain decoder"); + } + _binary_values.emplace_back(_data->data + _offset, length); + _offset += length; + } + return consumer.consume(_binary_values.data(), _binary_values.size()); +} + +Status ByteArrayPlainDecoder::skip_values(size_t num_values) { + for (int i = 0; i < num_values; ++i) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't skip enough bytes in plain decoder"); + } + _offset += length; + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h new file mode 100644 index 00000000000000..962981c8c77c8c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/types.h" +#include "format/format_common.h" +#include "format/parquet/parquet_common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_util.h" +#include "util/coding.h" +#include "util/slice.h" + +namespace doris { +template +class ColumnDecimal; +} // namespace doris + +namespace doris::format::parquet::native { +class ByteArrayPlainDecoder final : public Decoder { +public: + ByteArrayPlainDecoder() = default; + ~ByteArrayPlainDecoder() override = default; + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + +private: + std::vector _binary_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp new file mode 100644 index 00000000000000..6fda6d0d752a0a --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" + +#include + +#include "core/column/column_fixed_length_object.h" +#include "util/byte_stream_split.h" + +namespace doris::format::parquet::native { +Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet byte-stream-split decoder"); + } + DORIS_CHECK_EQ(_data->size % static_cast(_type_length), 0); + const int64_t stride = static_cast(_data->size / _type_length); + _decoded_values.resize(byte_size); + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + _offset / _type_length, num_values, stride, _decoded_values.data()); + _offset += byte_size; + return consumer.consume(_decoded_values.data(), num_values, static_cast(_type_length)); +} + +Status ByteStreamSplitDecoder::skip_values(size_t num_values) { + _offset += _type_length * num_values; + if (UNLIKELY(_offset > _data->size)) { + return Status::IOError( + "Out-of-bounds access in parquet data decoder: offset = {}, size = {}", _offset, + _data->size); + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h new file mode 100644 index 00000000000000..ae68f9c470b2bb --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { +class ByteStreamSplitDecoder final : public Decoder { +public: + ByteStreamSplitDecoder() = default; + ~ByteStreamSplitDecoder() override = default; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; + +private: + std::vector _decoded_values; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp new file mode 100644 index 00000000000000..c250c4bc18a955 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -0,0 +1,867 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_chunk_reader.h" + +#include +#include +#include + +#include +#include +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "core/column/column.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/data_type_serde.h" +#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/bit_util.h" +#include "util/block_compression.h" + +namespace cctz { +class time_zone; +} // namespace cctz +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { +namespace { + +Status translate_value_encoding(tparquet::Encoding::type encoding, + ParquetValueEncoding* translated) { + DORIS_CHECK(translated != nullptr); + switch (encoding) { + case tparquet::Encoding::PLAIN: + *translated = ParquetValueEncoding::PLAIN; + return Status::OK(); + case tparquet::Encoding::RLE_DICTIONARY: + case tparquet::Encoding::PLAIN_DICTIONARY: + *translated = ParquetValueEncoding::DICTIONARY; + return Status::OK(); + case tparquet::Encoding::RLE: + *translated = ParquetValueEncoding::RLE; + return Status::OK(); + case tparquet::Encoding::BIT_PACKED: + *translated = ParquetValueEncoding::BIT_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_BINARY_PACKED: + *translated = ParquetValueEncoding::DELTA_BINARY_PACKED; + return Status::OK(); + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_LENGTH_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::DELTA_BYTE_ARRAY: + *translated = ParquetValueEncoding::DELTA_BYTE_ARRAY; + return Status::OK(); + case tparquet::Encoding::BYTE_STREAM_SPLIT: + *translated = ParquetValueEncoding::BYTE_STREAM_SPLIT; + return Status::OK(); + default: + return Status::NotSupported("Unsupported Parquet encoding {}", + tparquet::to_string(encoding)); + } +} + +template +Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decoder& decoder, + const ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector, + int64_t* materialization_time) { + SCOPED_RAW_TIMER(materialization_time); + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + RETURN_IF_ERROR( + serde.read_column_from_parquet(column, decoder, context, run_length, state)); + break; + case ColumnSelectVector::NULL_DATA: + column.insert_many_defaults(run_length); + break; + case ColumnSelectVector::FILTERED_CONTENT: + RETURN_IF_ERROR(decoder.skip_values(run_length)); + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + return Status::OK(); +} + +} // namespace + +template +ColumnChunkReader::ColumnChunkReader( + io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows, + io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx) + : _field_schema(field_schema), + _max_rep_level(field_schema->repetition_level), + _max_def_level(field_schema->definition_level), + _stream_reader(reader), + _metadata(column_chunk->meta_data), + _offset_index(offset_index), + _total_rows(total_rows), + _io_ctx(io_ctx), + _page_read_ctx(page_read_ctx) {} + +template +Status ColumnChunkReader::init() { + size_t start_offset = has_dict_page(_metadata) ? _metadata.dictionary_page_offset + : _metadata.data_page_offset; + size_t chunk_size = _metadata.total_compressed_size; + // create page reader + _page_reader = create_page_reader( + _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata, + _page_read_ctx, _offset_index); + // get the block compression codec + RETURN_IF_ERROR(get_block_compression_codec(_metadata.codec, &_block_compress_codec)); + _state = INITIALIZED; + RETURN_IF_ERROR(_parse_first_page_header()); + return Status::OK(); +} + +template +Status ColumnChunkReader::skip_nested_values( + const std::vector& def_levels, size_t start_index) { + size_t no_value_cnt = 0; + size_t value_cnt = 0; + + DORIS_CHECK(start_index <= def_levels.size()); + for (size_t idx = start_index; idx < def_levels.size(); idx++) { + level_t def_level = def_levels[idx]; + if (IN_COLLECTION && def_level < _field_schema->repeated_parent_def_level) { + no_value_cnt++; + } else if (def_level < _field_schema->definition_level) { + no_value_cnt++; + } else { + value_cnt++; + } + } + + RETURN_IF_ERROR(skip_values(value_cnt, true)); + RETURN_IF_ERROR(skip_values(no_value_cnt, false)); + return Status::OK(); +} + +template +Status ColumnChunkReader::read_levels( + size_t num_values, std::vector* rep_levels, std::vector* def_levels) { + DORIS_CHECK(rep_levels != nullptr); + DORIS_CHECK(def_levels != nullptr); + if (_remaining_num_values < num_values || _remaining_rep_nums < num_values || + _remaining_def_nums < num_values) { + return Status::Corruption( + "Parquet level reader requested {} slots with only {}/{}/{} remaining", num_values, + _remaining_num_values, _remaining_rep_nums, _remaining_def_nums); + } + + const size_t start_index = def_levels->size(); + rep_levels->resize(rep_levels->size() + num_values, 0); + def_levels->resize(def_levels->size() + num_values, 0); + if (_max_rep_level > 0) { + const size_t decoded = _rep_level_decoder.get_levels( + rep_levels->data() + rep_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet repetition level stream ended after {} of {} slots", + decoded, num_values); + } + } + if (_max_def_level > 0) { + const size_t decoded = _def_level_decoder.get_levels( + def_levels->data() + def_levels->size() - num_values, num_values); + if (decoded != num_values) { + return Status::Corruption("Parquet definition level stream ended after {} of {} slots", + decoded, num_values); + } + } + _remaining_rep_nums -= num_values; + _remaining_def_nums -= num_values; + return skip_nested_values(*def_levels, start_index); +} + +template +Status ColumnChunkReader::_parse_first_page_header() { + RETURN_IF_ERROR(parse_page_header()); + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, + // so we should parse the directory page in next_page() + RETURN_IF_ERROR(_decode_dict_page()); + // parse the real first data page + RETURN_IF_ERROR(_page_reader->dict_next_page()); + _state = INITIALIZED; + } + + return Status::OK(); +} + +template +Status ColumnChunkReader::parse_page_header() { + if (_state == HEADER_PARSED || _state == DATA_LOADED) { + return Status::OK(); + } + RETURN_IF_ERROR(_page_reader->parse_page_header()); + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values + : header->data_page_header.num_values; + _remaining_rep_nums = page_num_values; + _remaining_def_nums = page_num_values; + _remaining_num_values = page_num_values; + + // no offset will parse all header. + if constexpr (OFFSET_INDEX == false) { + _chunk_parsed_values += _remaining_num_values; + } + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status ColumnChunkReader::next_page() { + _state = INITIALIZED; + RETURN_IF_ERROR(_page_reader->next_page()); + return Status::OK(); +} + +template +void ColumnChunkReader::_get_uncompressed_levels( + const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { + int32_t rl = page_v2.repetition_levels_byte_length; + int32_t dl = page_v2.definition_levels_byte_length; + _v2_rep_levels = Slice(page_data.data, rl); + _v2_def_levels = Slice(page_data.data + rl, dl); + page_data.data += dl + rl; + page_data.size -= dl + rl; +} + +template +Status ColumnChunkReader::load_page_data() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::Corruption("Should parse page header"); + } + + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + int32_t uncompressed_size = header->uncompressed_page_size; + bool page_loaded = false; + + // First, try to reuse a cache handle previously discovered by PageReader + // (header-only lookup) to avoid a second lookup here. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + size_t levels_size = 0; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + levels_size = rl + dl; + _v2_rep_levels = + Slice(reinterpret_cast(cached.data) + header_size, rl); + _v2_def_levels = + Slice(reinterpret_cast(cached.data) + header_size + rl, dl); + } + // payload_slice points to the bytes after header and levels + Slice payload_slice(cached.data + header_size + levels_size, + cached.size - header_size - levels_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + + if (cache_payload_is_decompressed) { + // Cached payload is already uncompressed + _page_data = payload_slice; + } else { + CHECK(_block_compress_codec); + // Decompress cached payload into _decompress_buf for decoding + size_t uncompressed_payload_size = + header->__isset.data_page_header_v2 + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->uncompressed_page_size); + _reserve_decompress_buf(uncompressed_payload_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_payload_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); + } + // page cache counters were incremented when PageReader did the header-only + // cache lookup. Do not increment again to avoid double-counting. + page_loaded = true; + } + } + + if (!page_loaded) { + if (_block_compress_codec != nullptr) { + Slice compressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + std::vector level_bytes; + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + // uncompressed_size = rl + dl + uncompressed_data_size + // compressed_size = rl + dl + compressed_data_size + uncompressed_size -= header_v2.repetition_levels_byte_length + + header_v2.definition_levels_byte_length; + // copy level bytes (rl + dl) so that we can cache header + levels + uncompressed payload + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), compressed_data.data, level_sz); + } + // now remove levels from compressed_data for decompression + _get_uncompressed_levels(header_v2, compressed_data); + } + bool is_v2_compressed = header->__isset.data_page_header_v2 && + header->data_page_header_v2.is_compressed; + bool page_has_compression = header->__isset.data_page_header || is_v2_compressed; + + if (page_has_compression) { + // Decompress payload for immediate decoding + _reserve_decompress_buf(uncompressed_size); + _page_data = Slice(_decompress_buf.get(), uncompressed_size); + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + _chunk_statistics.decompress_cnt++; + RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); + + // Decide whether to cache decompressed payload or compressed payload based on threshold + bool cache_payload_decompressed = should_cache_decompressed(header, _metadata); + + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + if (cache_payload_decompressed) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + // cache the compressed payload as-is (header | levels | compressed_payload) + _insert_page_into_cache( + level_bytes, Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + } else { + // no compression on this page, use the data directly + _page_data = Slice(compressed_data.data, compressed_data.size); + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } else { + // For uncompressed page, we may still need to extract v2 levels + std::vector level_bytes; + Slice uncompressed_data; + RETURN_IF_ERROR(_page_reader->get_page_data(uncompressed_data)); + if (header->__isset.data_page_header_v2) { + const tparquet::DataPageHeaderV2& header_v2 = header->data_page_header_v2; + size_t rl = header_v2.repetition_levels_byte_length; + size_t dl = header_v2.definition_levels_byte_length; + size_t level_sz = rl + dl; + if (level_sz > 0) { + level_bytes.resize(level_sz); + memcpy(level_bytes.data(), uncompressed_data.data, level_sz); + } + _get_uncompressed_levels(header_v2, uncompressed_data); + } + // copy page data out + _page_data = Slice(uncompressed_data.data, uncompressed_data.size); + // Optionally cache uncompressed data for uncompressed pages + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr) { + _insert_page_into_cache(level_bytes, _page_data); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Initialize repetition level and definition level. Skip when level = 0, which means required field. + if (_max_rep_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_rep_level_decoder.init_v2(_v2_rep_levels, _max_rep_level, + _remaining_rep_nums)); + } else { + RETURN_IF_ERROR(_rep_level_decoder.init( + &_page_data, header->data_page_header.repetition_level_encoding, _max_rep_level, + _remaining_rep_nums)); + } + } + if (_max_def_level > 0) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_level_time); + if (header->__isset.data_page_header_v2) { + RETURN_IF_ERROR(_def_level_decoder.init_v2(_v2_def_levels, _max_def_level, + _remaining_def_nums)); + } else { + RETURN_IF_ERROR(_def_level_decoder.init( + &_page_data, header->data_page_header.definition_level_encoding, _max_def_level, + _remaining_def_nums)); + } + } + auto encoding = header->__isset.data_page_header_v2 ? header->data_page_header_v2.encoding + : header->data_page_header.encoding; + // change the deprecated encoding to RLE_DICTIONARY + if (encoding == tparquet::Encoding::PLAIN_DICTIONARY) { + encoding = tparquet::Encoding::RLE_DICTIONARY; + } + _current_encoding = encoding; + + // Reuse page decoder + if (_decoders.find(static_cast(encoding)) != _decoders.end()) { + _page_decoder = _decoders[static_cast(encoding)].get(); + } else { + std::unique_ptr page_decoder; + RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + _decoders[static_cast(encoding)] = std::move(page_decoder); + _page_decoder = _decoders[static_cast(encoding)].get(); + } + RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); + + _state = DATA_LOADED; + return Status::OK(); +} + +template +Status ColumnChunkReader::_decode_dict_page() { + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + DCHECK_EQ(tparquet::PageType::DICTIONARY_PAGE, header->type); + SCOPED_RAW_TIMER(&_chunk_statistics.decode_dict_time); + + // Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. + // Prefer using RLE_DICTIONARY in a data page and PLAIN in a dictionary page for Parquet 2.0+ files. + // refer: https://github.com/apache/parquet-format/blob/master/Encodings.md + tparquet::Encoding::type dict_encoding = header->dictionary_page_header.encoding; + if (dict_encoding != tparquet::Encoding::PLAIN_DICTIONARY && + dict_encoding != tparquet::Encoding::PLAIN) { + return Status::InternalError("Unsupported dictionary encoding {}", + tparquet::to_string(dict_encoding)); + } + + // Prepare dictionary data + int32_t uncompressed_size = header->uncompressed_page_size; + auto dict_data = make_unique_buffer(uncompressed_size); + bool dict_loaded = false; + + // Try to load dictionary page from cache + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + if (_page_reader->has_page_cache_handle()) { + const PageCacheHandle& handle = _page_reader->page_cache_handle(); + Slice cached = handle.data(); + size_t header_size = _page_reader->header_bytes().size(); + // Dictionary page layout in cache: header | payload (compressed or uncompressed) + Slice payload_slice(cached.data + header_size, cached.size - header_size); + + bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + + if (cache_payload_is_decompressed) { + // Use cached decompressed dictionary data + memcpy(dict_data.get(), payload_slice.data, payload_slice.size); + dict_loaded = true; + } else { + CHECK(_block_compress_codec); + // Decompress cached compressed dictionary data + Slice dict_slice(dict_data.get(), uncompressed_size); + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + dict_loaded = true; + } + + // When dictionary page is loaded from cache, we need to skip the page data + // to update the offset correctly (similar to calling get_page_data()) + if (dict_loaded) { + _page_reader->skip_page_data(); + } + } + } + + if (!dict_loaded) { + // Load and decompress dictionary page from file + if (_block_compress_codec != nullptr) { + auto dict_num = header->dictionary_page_header.num_values; + if (dict_num == 0 && uncompressed_size != 0) { + return Status::IOError( + "Dictionary page's num_values is {} but uncompressed_size is {}", dict_num, + uncompressed_size); + } + Slice compressed_data; + Slice dict_slice(dict_data.get(), uncompressed_size); + if (dict_num != 0) { + RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); + RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &dict_slice)); + } + + // Decide whether to cache decompressed or compressed dictionary based on threshold + // If uncompressed_page_size == 0, should_cache_decompressed will return true + bool cache_payload_decompressed = should_cache_decompressed(header, _metadata); + + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; // Dictionary pages don't have levels + if (cache_payload_decompressed) { + // Cache the decompressed dictionary page + // If dict_num == 0, `dict_slice` will be empty + _insert_page_into_cache(empty_levels, dict_slice); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } else { + if (config::enable_parquet_cache_compressed_pages) { + DCHECK(!compressed_data.empty()); + // Cache the compressed dictionary page + _insert_page_into_cache(empty_levels, + Slice(compressed_data.data, compressed_data.size)); + _chunk_statistics.page_cache_compressed_write_counter += 1; + } + } + } + // `get_page_data` not called, we should skip the page data + // Because `_insert_page_into_cache` will use _page_reader, we should exec `skip_page_data` after `_insert_page_into_cache` + if (dict_num == 0) { + _page_reader->skip_page_data(); + } + } else { + Slice dict_slice; + RETURN_IF_ERROR(_page_reader->get_page_data(dict_slice)); + // The data is stored by BufferedStreamReader, we should copy it out + memcpy(dict_data.get(), dict_slice.data, dict_slice.size); + + // Cache the uncompressed dictionary page + if (_page_read_ctx.enable_parquet_file_page_cache && + !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && + !_page_reader->header_bytes().empty()) { + std::vector empty_levels; + Slice payload(dict_data.get(), uncompressed_size); + _insert_page_into_cache(empty_levels, payload); + _chunk_statistics.page_cache_decompressed_write_counter += 1; + } + } + } + + // Cache page decoder + std::unique_ptr page_decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(_metadata.type, tparquet::Encoding::RLE_DICTIONARY, page_decoder)); + // Set type length + page_decoder->set_type_length(_get_type_length()); + // Set the dictionary data + RETURN_IF_ERROR(page_decoder->set_dict(dict_data, uncompressed_size, + header->dictionary_page_header.num_values)); + _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)] = std::move(page_decoder); + + _has_dict = true; + return Status::OK(); +} + +template +void ColumnChunkReader::_reserve_decompress_buf(size_t size) { + if (size > _decompress_buf_size) { + _decompress_buf_size = BitUtil::next_power_of_two(size); + _decompress_buf = make_unique_buffer(_decompress_buf_size); + } +} + +template +void ColumnChunkReader::_insert_page_into_cache( + const std::vector& level_bytes, const Slice& payload) { + StoragePageCache::CacheKey key = + _page_reader->make_page_cache_key(_page_reader->header_start_offset()); + const std::vector& header_bytes = _page_reader->header_bytes(); + size_t total = header_bytes.size() + level_bytes.size() + payload.size; + auto page = std::make_unique(total, true, segment_v2::DATA_PAGE); + size_t pos = 0; + memcpy(page->data() + pos, header_bytes.data(), header_bytes.size()); + pos += header_bytes.size(); + if (!level_bytes.empty()) { + memcpy(page->data() + pos, level_bytes.data(), level_bytes.size()); + pos += level_bytes.size(); + } + if (payload.size > 0) { + memcpy(page->data() + pos, payload.data, payload.size); + pos += payload.size; + } + page->reset_size(total); + PageCacheHandle handle; + StoragePageCache::instance()->insert(key, page.get(), &handle, segment_v2::DATA_PAGE); + page.release(); + _chunk_statistics.page_cache_write_counter += 1; +} + +template +Status ColumnChunkReader::skip_values(size_t num_values, + bool skip_data) { + if (UNLIKELY(_remaining_num_values < num_values)) { + return Status::IOError("Skip too many values in current page. {} vs. {}", + _remaining_num_values, num_values); + } + _remaining_num_values -= num_values; + if (skip_data) { + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + return _page_decoder->skip_values(num_values); + } else { + return Status::OK(); + } +} + +template +Status ColumnChunkReader::materialize_values( + MutableColumnPtr& doris_column, const DataTypeSerDe& serde, ParquetDecodeContext& context, + ParquetMaterializationState& state, ColumnSelectVector& select_vector) { + if (select_vector.num_values() == 0) { + return Status::OK(); + } + SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + if (UNLIKELY((doris_column->is_column_dictionary() || context.dictionary_index_only) && + !_has_dict)) { + return Status::IOError("Not dictionary coded"); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + _remaining_num_values -= select_vector.num_values(); + RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); + if (select_vector.has_filter()) { + return decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, &_chunk_statistics.materialization_time); + } + return decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, &_chunk_statistics.materialization_time); +} + +template +Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { + if constexpr (OFFSET_INDEX) { + while (true) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + break; + } else if (has_next_page()) { + RETURN_IF_ERROR(next_page()); + _current_row = _page_reader->start_row(); + } else [[unlikely]] { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + }; + + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + } else { + while (true) { + RETURN_IF_ERROR(parse_page_header()); + if (_page_reader->is_header_v2() || !IN_COLLECTION) { + if (_page_reader->start_row() <= left_row && left_row < _page_reader->end_row()) { + RETURN_IF_ERROR(load_page_data()); + // this page contain this row. + RETURN_IF_ERROR(_skip_nested_rows_in_page(left_row - _current_row)); + _current_row = left_row; + break; + } + + _current_row = _page_reader->end_row(); + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", left_row, + _current_row); + } + } else { + RETURN_IF_ERROR(load_page_data()); + std::vector rep_levels; + std::vector def_levels; + bool cross_page = false; + + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, left_row - _current_row, + &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + bool need_load_next_page = true; + while (cross_page) { + need_load_next_page = false; + rep_levels.clear(); + def_levels.clear(); + RETURN_IF_ERROR(load_cross_page_nested_row(rep_levels, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + } + if (left_row == _current_row) { + break; + } + if (need_load_next_page) { + if (has_next_page()) [[likely]] { + RETURN_IF_ERROR(next_page()); + } else { + return Status::InternalError("no match seek row {}, current row {}", + left_row, _current_row); + } + } + } + }; + } + + return Status::OK(); +} + +template +Status ColumnChunkReader::_skip_nested_rows_in_page(size_t num_rows) { + if (num_rows == 0) { + return Status::OK(); + } + + std::vector rep_levels; + std::vector def_levels; + + bool cross_page = false; + size_t result_rows = 0; + RETURN_IF_ERROR(load_page_nested_rows(rep_levels, num_rows, &result_rows, &cross_page)); + RETURN_IF_ERROR(fill_def(def_levels)); + RETURN_IF_ERROR(skip_nested_values(def_levels)); + DCHECK(cross_page == false); + if (num_rows != result_rows) [[unlikely]] { + return Status::InternalError("no match skip rows, expect {} vs. real {}", num_rows, + result_rows); + } + return Status::OK(); +} + +template +Status ColumnChunkReader::load_page_nested_rows( + std::vector& rep_levels, size_t max_rows, size_t* result_rows, bool* cross_page) { + if (_state != DATA_LOADED) [[unlikely]] { + return Status::IOError("Should load page data first to load nested rows"); + } + *cross_page = false; + *result_rows = 0; + rep_levels.reserve(rep_levels.size() + _remaining_rep_nums); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (rep_level == 0) { // rep_level 0 indicates start of new row + if (*result_rows == max_rows) { // this page contain max_rows, page no end. + _current_row += max_rows; + _rep_level_rewind_one(); + return Status::OK(); + } + (*result_rows)++; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + _current_row += *result_rows; + + auto need_check_cross_page = [&]() -> bool { + return !OFFSET_INDEX && IN_COLLECTION && _remaining_rep_nums == 0 && + !_page_reader->is_header_v2() && has_next_page(); + }; + *cross_page = need_check_cross_page(); + return Status::OK(); +}; + +template +Status ColumnChunkReader::load_cross_page_nested_row( + std::vector& rep_levels, bool* cross_page) { + RETURN_IF_ERROR(next_page()); + RETURN_IF_ERROR(parse_page_header()); + RETURN_IF_ERROR(load_page_data()); + + *cross_page = has_next_page(); + while (_remaining_rep_nums) { + level_t rep_level = _rep_level_get_next(); + if (rep_level == 0) { // rep_level 0 indicates start of new row + *cross_page = false; + _rep_level_rewind_one(); + break; + } + _remaining_rep_nums--; + rep_levels.emplace_back(rep_level); + } + return Status::OK(); +} + +template +int32_t ColumnChunkReader::_get_type_length() { + switch (_field_schema->physical_type) { + case tparquet::Type::INT32: + [[fallthrough]]; + case tparquet::Type::FLOAT: + return 4; + case tparquet::Type::INT64: + [[fallthrough]]; + case tparquet::Type::DOUBLE: + return 8; + case tparquet::Type::INT96: + return 12; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + return _field_schema->parquet_schema.type_length; + default: + return -1; + } +} + +/** + * Checks if the given column has a dictionary page. + * + * This function determines the presence of a dictionary page by checking the + * dictionary_page_offset field in the column metadata. The dictionary_page_offset + * must be set and greater than 0, and it must be less than the data_page_offset. + * + * The reason for these checks is based on the implementation in the Java version + * of ORC, where dictionary_page_offset is used to indicate the absence of a dictionary. + * Additionally, Parquet may write an empty row group, in which case the dictionary page + * content would be empty, and thus the dictionary page should not be read. + * + * See https://github.com/apache/arrow/pull/2667/files + */ +bool has_dict_page(const tparquet::ColumnMetaData& column) { + return column.__isset.dictionary_page_offset && column.dictionary_page_offset > 0 && + column.dictionary_page_offset < column.data_page_offset; +} + +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; +template class ColumnChunkReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h new file mode 100644 index 00000000000000..653a7cc19cedd8 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -0,0 +1,290 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_string.h" +#include "core/data_type/data_type.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "format/parquet/parquet_common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "util/slice.h" + +namespace doris { +class BlockCompressionCodec; +class DataTypeSerDe; +struct FieldSchema; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris::format::parquet::native { +using ::doris::FieldSchema; +using ::doris::ColumnString; + +struct ColumnChunkReaderStatistics { + int64_t decompress_time = 0; + int64_t decompress_cnt = 0; + int64_t decode_header_time = 0; + int64_t decode_value_time = 0; + int64_t materialization_time = 0; + int64_t decode_dict_time = 0; + int64_t decode_level_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_read_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; +}; + +/** + * Read and decode parquet column data into doris block column. + *

Usage:

+ * // Create chunk reader + * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, + * tparquet::ColumnChunk* column_chunk, + * FieldSchema* fieldSchema); + * // Initialize chunk reader + * chunk_reader.init(); + * while (chunk_reader.has_next_page()) { + * // Seek to next page header. Only read and parse the page header, not page data. + * chunk_reader.next_page(); + * // Load data to decoder. Load the page data into underlying container. + * // Or, we can call the chunk_reader.skip_page() to skip current page. + * chunk_reader.load_page_data(); + * // Decode values into column or slice. + * // Or, we can call chunk_reader.skip_values(num_values) to skip some values. + * chunk_reader.materialize_values(column, serde, context, state, selection); + * } + */ +template +class ColumnChunkReader { +public: + ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, + FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_row, io::IOContext* io_ctx, + const ParquetPageReadContext& page_read_ctx); + ~ColumnChunkReader() = default; + + // Initialize chunk reader, will generate the decoder and codec. + Status init(); + + // Whether the chunk reader has a more page to read. + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + return _page_reader->has_next_page(); + } else { + // no offset need parse all page header. + return _chunk_parsed_values < _metadata.num_values; + } + } + + // Skip some values(will not read and parse) in current page if the values are filtered by predicates. + // when skip_data = false, the underlying decoder will not skip data, + // only used when maintaining the consistency of _remaining_num_values. + Status skip_values(size_t num_values, bool skip_data = true); + + // Load page data into the underlying container, + // and initialize the repetition and definition level decoder for current page data. + Status load_page_data(); + Status load_page_data_idempotent() { + if (_state == DATA_LOADED) { + return Status::OK(); + } + return load_page_data(); + } + // The remaining number of values in current page(including null values). Decreased when reading or skipping. + uint32_t remaining_num_values() const { return _remaining_num_values; } + + // Apply one logical selection to the encoded page. Decoder advances physical payload cursors; + // SerDe interprets each selected raw span and materializes the destination Doris column. + Status materialize_values(MutableColumnPtr& doris_column, const DataTypeSerDe& serde, + ParquetDecodeContext& context, ParquetMaterializationState& state, + ColumnSelectVector& select_vector); + + // Get the repetition level decoder of current page. + LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } + // Get the definition level decoder of current page. + LevelDecoder& def_level_decoder() { return _def_level_decoder; } + + level_t max_rep_level() const { return _max_rep_level; } + level_t max_def_level() const { return _max_def_level; } + + bool has_dict() const { return _has_dict; }; + + // Get page decoder + Decoder* get_page_decoder() { return _page_decoder; } + + tparquet::Encoding::type current_encoding() const { return _current_encoding; } + + ColumnChunkReaderStatistics& chunk_statistics() { + _chunk_statistics.decode_header_time = _page_reader->page_statistics().decode_header_time; + _chunk_statistics.skip_page_header_num = + _page_reader->page_statistics().skip_page_header_num; + _chunk_statistics.parse_page_header_num = + _page_reader->page_statistics().parse_page_header_num; + _chunk_statistics.read_page_header_time = + _page_reader->page_statistics().read_page_header_time; + // PageReader statistics are already cumulative. Snapshot them instead of folding the same + // totals into the chunk on every FileScannerV2 batch. Cache write counters are owned by + // ColumnChunkReader because insertion happens after decompression and therefore remain in + // the chunk accumulator above. + _chunk_statistics.page_read_counter = _page_reader->page_statistics().page_read_counter; + _chunk_statistics.page_cache_hit_counter = + _page_reader->page_statistics().page_cache_hit_counter; + _chunk_statistics.page_cache_missing_counter = + _page_reader->page_statistics().page_cache_missing_counter; + _chunk_statistics.page_cache_compressed_hit_counter = + _page_reader->page_statistics().page_cache_compressed_hit_counter; + _chunk_statistics.page_cache_decompressed_hit_counter = + _page_reader->page_statistics().page_cache_decompressed_hit_counter; + return _chunk_statistics; + } + + Decoder* dictionary_decoder() { + return _decoders[static_cast(tparquet::Encoding::RLE_DICTIONARY)].get(); + } + + size_t page_start_row() const { return _page_reader->start_row(); } + + size_t page_end_row() const { return _page_reader->end_row(); } + + Status parse_page_header(); + Status next_page(); + + Status seek_to_nested_row(size_t left_row); + // Decode level slots without materializing their values. `read_levels` is the flat-column + // counterpart of load_page_nested_rows()+fill_def(): both advance definition/repetition and + // value streams together so a levels-only consumer cannot desynchronize the next data page. + Status read_levels(size_t num_values, std::vector* rep_levels, + std::vector* def_levels); + Status skip_nested_values(const std::vector& def_levels, size_t start_index = 0); + Status fill_def(std::vector& def_values) { + auto before_sz = def_values.size(); + auto append_sz = _remaining_def_nums - _remaining_rep_nums; + def_values.resize(before_sz + append_sz, 0); + if (max_def_level() != 0) { + auto ptr = def_values.data() + before_sz; + _def_level_decoder.get_levels(ptr, append_sz); + } + _remaining_def_nums -= append_sz; + return Status::OK(); + } + + Status load_page_nested_rows(std::vector& rep_levels, size_t max_rows, + size_t* result_rows, bool* cross_page); + Status load_cross_page_nested_row(std::vector& rep_levels, bool* cross_page); + + Slice get_page_data() const { return _page_data; } + const Slice& v2_rep_levels() const { return _v2_rep_levels; } + const Slice& v2_def_levels() const { return _v2_def_levels; } + ColumnChunkReaderStatistics& statistics() { return chunk_statistics(); } + +private: + enum ColumnChunkReaderState { NOT_INIT, INITIALIZED, HEADER_PARSED, DATA_LOADED, PAGE_SKIPPED }; + + // for check dict page. + Status _parse_first_page_header(); + Status _decode_dict_page(); + + void _reserve_decompress_buf(size_t size); + int32_t _get_type_length(); + void _insert_page_into_cache(const std::vector& level_bytes, const Slice& payload); + + void _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _skip_nested_rows_in_page(size_t num_rows); + + level_t _rep_level_get_next() { + if constexpr (IN_COLLECTION) { + return _rep_level_decoder.get_next(); + } + return 0; + } + + void _rep_level_rewind_one() { + if constexpr (IN_COLLECTION) { + _rep_level_decoder.rewind_one(); + } + } + + ColumnChunkReaderState _state = NOT_INIT; + FieldSchema* _field_schema = nullptr; + const level_t _max_rep_level; + const level_t _max_def_level; + + io::BufferedStreamReader* _stream_reader = nullptr; + tparquet::ColumnMetaData _metadata; + const tparquet::OffsetIndex* _offset_index = nullptr; + size_t _current_row = 0; + size_t _total_rows = 0; + io::IOContext* _io_ctx = nullptr; + + std::unique_ptr> _page_reader; + BlockCompressionCodec* _block_compress_codec = nullptr; + + ParquetPageReadContext _page_read_ctx; + + LevelDecoder _rep_level_decoder; + LevelDecoder _def_level_decoder; + size_t _chunk_parsed_values = 0; + // this page remaining rep/def nums + // if max_rep_level = 0 / max_def_level = 0, this value retail hava value. + uint32_t _remaining_rep_nums = 0; + uint32_t _remaining_def_nums = 0; + // this page remaining values to be processed (for read/skip). + // need parse this page header. + uint32_t _remaining_num_values = 0; + + Slice _page_data; + DorisUniqueBufferPtr _decompress_buf; + size_t _decompress_buf_size = 0; + Slice _v2_rep_levels; + Slice _v2_def_levels; + bool _dict_checked = false; + bool _has_dict = false; + Decoder* _page_decoder = nullptr; + tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; + // Map: encoding -> Decoder + // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding + std::unordered_map> _decoders; + ColumnChunkReaderStatistics _chunk_statistics; +}; + +bool has_dict_page(const tparquet::ColumnMetaData& column); + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp new file mode 100644 index 00000000000000..927ba73866c4c8 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -0,0 +1,1340 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/column_reader.h" + +#include +#include +#include + +#include +#include + +#include "common/status.h" +#include "core/column/column.h" +#include "core/column/column_array.h" +#include "core/column/column_map.h" +#include "core/column/column_nullable.h" +#include "core/column/column_struct.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "io/fs/tracing_file_reader.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::parquet::native { +namespace { + +ParquetTimeUnit parquet_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; + } + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; + } + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; + } + return ParquetTimeUnit::UNKNOWN; +} + +bool is_direct_integer_type(PrimitiveType type) { + switch (type) { + case TYPE_TINYINT: + case TYPE_SMALLINT: + case TYPE_INT: + case TYPE_BIGINT: + case TYPE_LARGEINT: + return true; + default: + return false; + } +} + +bool is_direct_decimal_type(PrimitiveType type) { + switch (type) { + case TYPE_DECIMALV2: + case TYPE_DECIMAL32: + case TYPE_DECIMAL64: + case TYPE_DECIMAL128I: + case TYPE_DECIMAL256: + return true; + default: + return false; + } +} + +// The target SerDe can fuse physical decode with these logical type changes. Less common schema +// changes retain the generic file-format converter as a compatibility path: the decoder still +// exposes raw spans, but the source SerDe first materializes a reusable source column before the +// generic logical cast. Ordinary scans and numeric widening never allocate that source column. +bool serde_can_materialize_directly(const DataTypePtr& source_type, + const DataTypePtr& target_type) { + const auto source = remove_nullable(source_type)->get_primitive_type(); + const auto target = remove_nullable(target_type)->get_primitive_type(); + return source == target || (is_direct_integer_type(source) && is_direct_integer_type(target)) || + (source == TYPE_FLOAT && target == TYPE_DOUBLE) || + (is_direct_decimal_type(source) && is_direct_decimal_type(target)) || + (is_string_type(source) && is_string_type(target)); +} + +Status init_decode_context(const FieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + DORIS_CHECK(context != nullptr); + switch (field.physical_type) { + case tparquet::Type::BOOLEAN: + context->physical_type = ParquetPhysicalType::BOOLEAN; + break; + case tparquet::Type::INT32: + context->physical_type = ParquetPhysicalType::INT32; + break; + case tparquet::Type::INT64: + context->physical_type = ParquetPhysicalType::INT64; + break; + case tparquet::Type::INT96: + context->physical_type = ParquetPhysicalType::INT96; + break; + case tparquet::Type::FLOAT: + context->physical_type = ParquetPhysicalType::FLOAT; + break; + case tparquet::Type::DOUBLE: + context->physical_type = ParquetPhysicalType::DOUBLE; + break; + case tparquet::Type::BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::BYTE_ARRAY; + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + context->physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY; + break; + default: + return Status::NotSupported("Unsupported Parquet physical type {}", + tparquet::to_string(field.physical_type)); + } + + const auto& schema = field.parquet_schema; + context->type_length = schema.__isset.type_length ? schema.type_length : -1; + context->decimal_precision = schema.__isset.precision ? schema.precision : -1; + context->decimal_scale = schema.__isset.scale ? schema.scale : -1; + context->timezone = ctz; + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + context->logical_type = ParquetLogicalType::STRING; + } else if (logical.__isset.DECIMAL) { + context->logical_type = ParquetLogicalType::DECIMAL; + context->decimal_precision = logical.DECIMAL.precision; + context->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.DATE) { + context->logical_type = ParquetLogicalType::DATE; + } else if (logical.__isset.TIME) { + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = parquet_time_unit(logical.TIME.unit); + } else if (logical.__isset.TIMESTAMP) { + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = parquet_time_unit(logical.TIMESTAMP.unit); + context->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + } else if (logical.__isset.INTEGER) { + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_bit_width = logical.INTEGER.bitWidth; + context->logical_integer_is_signed = logical.INTEGER.isSigned; + } else if (logical.__isset.UUID) { + context->logical_type = ParquetLogicalType::UUID; + context->logical_uuid = true; + } else if (logical.__isset.FLOAT16) { + context->logical_type = ParquetLogicalType::FLOAT16; + context->logical_float16 = true; + } + if (context->logical_uuid && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 16)) { + return Status::Corruption("Parquet UUID field {} must be FIXED_LEN_BYTE_ARRAY(16)", + field.name); + } + if (context->logical_float16 && + (context->physical_type != ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY || + context->type_length != 2)) { + return Status::Corruption("Parquet FLOAT16 field {} must be FIXED_LEN_BYTE_ARRAY(2)", + field.name); + } + return Status::OK(); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + context->logical_type = ParquetLogicalType::STRING; + break; + case tparquet::ConvertedType::DECIMAL: + context->logical_type = ParquetLogicalType::DECIMAL; + break; + case tparquet::ConvertedType::DATE: + context->logical_type = ParquetLogicalType::DATE; + break; + case tparquet::ConvertedType::TIME_MILLIS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MILLIS; + break; + case tparquet::ConvertedType::TIME_MICROS: + context->logical_type = ParquetLogicalType::TIME; + context->time_unit = ParquetTimeUnit::MICROS; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MILLIS; + break; + case tparquet::ConvertedType::TIMESTAMP_MICROS: + context->logical_type = ParquetLogicalType::TIMESTAMP; + context->time_unit = ParquetTimeUnit::MICROS; + break; + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::INT_64: + context->logical_type = ParquetLogicalType::INTEGER; + context->logical_integer_is_signed = + schema.converted_type >= tparquet::ConvertedType::INT_8; + context->logical_integer_bit_width = + schema.converted_type == tparquet::ConvertedType::UINT_8 || + schema.converted_type == tparquet::ConvertedType::INT_8 + ? 8 + : schema.converted_type == tparquet::ConvertedType::UINT_16 || + schema.converted_type == tparquet::ConvertedType::INT_16 + ? 16 + : schema.converted_type == tparquet::ConvertedType::UINT_32 || + schema.converted_type == tparquet::ConvertedType::INT_32 + ? 32 + : 64; + break; + default: + break; + } + return Status::OK(); +} + +} // namespace + +static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, + const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = def_levels.size(); + DCHECK_EQ(num_levels, rep_levels.size()); + size_t origin_size = null_map.size(); + null_map.resize(origin_size + num_levels); + size_t pos = origin_size; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (def_levels[i] >= field->definition_level) { + null_map[pos++] = 0; + } else { + null_map[pos++] = 1; + } + } + null_map.resize(pos); +} + +static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offsets_data, + NullMap* null_map_ptr, const std::vector& rep_levels, + const std::vector& def_levels) { + size_t num_levels = rep_levels.size(); + DCHECK_EQ(num_levels, def_levels.size()); + size_t origin_size = offsets_data.size(); + offsets_data.resize(origin_size + num_levels); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(origin_size + num_levels); + } + size_t offset_pos = origin_size - 1; + for (size_t i = 0; i < num_levels; ++i) { + // skip the levels affect its ancestor or its descendants + if (def_levels[i] < field->repeated_parent_def_level || + rep_levels[i] > field->repetition_level) { + continue; + } + if (rep_levels[i] == field->repetition_level) { + offsets_data[offset_pos]++; + continue; + } + offset_pos++; + offsets_data[offset_pos] = offsets_data[offset_pos - 1]; + if (def_levels[i] >= field->definition_level) { + offsets_data[offset_pos]++; + } + if (null_map_ptr != nullptr) { + if (def_levels[i] >= field->definition_level - 1) { + (*null_map_ptr)[offset_pos] = 0; + } else { + (*null_map_ptr)[offset_pos] = 1; + } + } + } + offsets_data.resize(offset_pos + 1); + if (null_map_ptr != nullptr) { + null_map_ptr->resize(offset_pos + 1); + } +} + +Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection, + const std::set& column_ids, + const std::set& filter_column_ids) { + size_t total_rows = row_group.num_rows; + if (field->data_type->get_primitive_type() == TYPE_ARRAY) { + std::unique_ptr element_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + element_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids)); + auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + element_reader->set_column_in_nested(); + RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); + array_reader->_filter_column_ids = filter_column_ids; + reader.reset(array_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_MAP) { + std::unique_ptr key_reader; + std::unique_ptr value_reader; + + if (column_ids.empty() || + column_ids.find(field->children[0].get_column_id()) != column_ids.end()) { + // Create key reader + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + key_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[0]); + key_reader = std::move(skip_reader); + } + + if (column_ids.empty() || + column_ids.find(field->children[1].get_column_id()) != column_ids.end()) { + // Create value reader + RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, + value_reader, max_buf_size, col_offsets, state, true, column_ids, + filter_column_ids)); + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &field->children[1]); + value_reader = std::move(skip_reader); + } + + auto map_reader = MapColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + key_reader->set_column_in_nested(); + value_reader->set_column_in_nested(); + RETURN_IF_ERROR(map_reader->init(std::move(key_reader), std::move(value_reader), field)); + map_reader->_filter_column_ids = filter_column_ids; + reader.reset(map_reader.release()); + } else if (field->data_type->get_primitive_type() == TYPE_STRUCT) { + std::unordered_map> child_readers; + child_readers.reserve(field->children.size()); + int non_skip_reader_idx = -1; + for (int i = 0; i < field->children.size(); ++i) { + auto& child = field->children[i]; + std::unique_ptr child_reader; + if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) { + RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, + in_collection, column_ids, filter_column_ids)); + child_readers[child.name] = std::move(child_reader); + // Record the first non-SkippingReader + if (non_skip_reader_idx == -1) { + non_skip_reader_idx = i; + } + } else { + auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, + io_ctx, &child); + skip_reader->_filter_column_ids = filter_column_ids; + child_readers[child.name] = std::move(skip_reader); + } + child_readers[child.name]->set_column_in_nested(); + } + // If all children are SkipReadingReader, force the first child to call create + if (non_skip_reader_idx == -1) { + std::unique_ptr child_reader; + RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, + child_reader, max_buf_size, col_offsets, state, in_collection, + column_ids, filter_column_ids)); + child_reader->set_column_in_nested(); + child_readers[field->children[0].name] = std::move(child_reader); + } + auto struct_reader = StructColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); + RETURN_IF_ERROR(struct_reader->init(std::move(child_readers), field)); + struct_reader->_filter_column_ids = filter_column_ids; + reader.reset(struct_reader.release()); + } else { + auto physical_index = field->physical_column_index; + const tparquet::OffsetIndex* offset_index = + col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index] + : nullptr; + + const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + if (in_collection) { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } else { + if (offset_index == nullptr) { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } else { + auto scalar_reader = ScalarColumnReader::create_unique( + row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); + + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + scalar_reader->_filter_column_ids = filter_column_ids; + reader.reset(scalar_reader.release()); + } + } + } + return Status::OK(); +} + +void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const { + result_ranges->add(page_row_range); + RowRanges::ranges_intersection(*result_ranges, _row_ranges, result_ranges); +} + +template +Status ScalarColumnReader::init(io::FileReaderSPtr file, + FieldSchema* field, + size_t max_buf_size, + RuntimeState* state) { + _field_schema = field; + auto& chunk_meta = _chunk_meta.meta_data; + int64_t chunk_start = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset + : chunk_meta.data_page_offset; + size_t chunk_len = chunk_meta.total_compressed_size; + size_t prefetch_buffer_size = std::min(chunk_len, max_buf_size); + if ((typeid_cast(file.get()) && + typeid_cast( + ((doris::io::TracingFileReader*)(file.get()))->inner_reader().get())) || + typeid_cast(file.get())) { + // turn off prefetch data when using MergeRangeFileReader + prefetch_buffer_size = 0; + } + _stream_reader = std::make_unique(file, chunk_start, chunk_len, + prefetch_buffer_size); + ParquetPageReadContext ctx( + (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache); + + _chunk_reader = std::make_unique>( + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx); + RETURN_IF_ERROR(_chunk_reader->init()); + RETURN_IF_ERROR(init_decode_context(*field, _ctz, &_decode_context)); + return Status::OK(); +} + +template +Status ScalarColumnReader::_skip_values(size_t num_values) { + if (num_values == 0) { + return Status::OK(); + } + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t skipped = 0; + size_t null_size = 0; + size_t nonnull_size = 0; + while (skipped < num_values) { + level_t def_level = -1; + size_t loop_skip = def_decoder.get_next_run(&def_level, num_values - skipped); + if (loop_skip == 0) { + std::stringstream ss; + auto& bit_reader = def_decoder.rle_decoder().bit_reader(); + ss << "def_decoder buffer (hex): "; + for (size_t i = 0; i < bit_reader.max_bytes(); ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') + << static_cast(bit_reader.buffer()[i]) << " "; + } + LOG(WARNING) << ss.str(); + return Status::InternalError("Failed to decode definition level."); + } + if (def_level < _field_schema->definition_level) { + null_size += loop_skip; + } else { + nonnull_size += loop_skip; + } + skipped += loop_skip; + } + if (null_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(null_size, false)); + } + if (nonnull_size > 0) { + RETURN_IF_ERROR(_chunk_reader->skip_values(nonnull_size, true)); + } + } else { + RETURN_IF_ERROR(_chunk_reader->skip_values(num_values)); + } + return Status::OK(); +} + +template +Status ScalarColumnReader::_read_values(size_t num_values, + ColumnPtr& doris_column, + const DataTypePtr& type, + FilterMap& filter_map, + bool is_dict_filter) { + if (num_values == 0) { + return Status::OK(); + } + MutableColumnPtr data_column; + _null_run_lengths.clear(); + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level; + size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + std::stringstream ss; + auto& bit_reader = def_decoder.rle_decoder().bit_reader(); + ss << "def_decoder buffer (hex): "; + for (size_t i = 0; i < bit_reader.max_bytes(); ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') + << static_cast(bit_reader.buffer()[i]) << " "; + } + LOG(WARNING) << ss.str(); + return Status::InternalError("Failed to decode definition level."); + } + + bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + prev_is_null = is_null; + has_read += loop_read; + } + } + } else { + if (_chunk_reader->max_def_level() > 0) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (_null_run_lengths.empty()) { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back((u_short)remaining); + } + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, map_data_column, + &filter_map, _filter_map_index)); + _filter_map_index += num_values; + } + DORIS_CHECK(_serde != nullptr); + return _chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector); +} + +/** + * Load the nested column data of complex type. + * A row of complex type may be stored across two(or more) pages, and the parameter `align_rows` indicates that + * whether the reader should read the remaining value of the last row in previous page. + */ +template +Status ScalarColumnReader::_read_nested_column( + ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter) { + _rep_levels.clear(); + _def_levels.clear(); + + // Handle nullable columns + MutableColumnPtr data_column; + NullMap* map_data_column = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + SCOPED_RAW_TIMER(&_decode_null_map_time); + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + data_column = nullable_column->get_nested_column_ptr(); + map_data_column = &(nullable_column->get_null_map_data()); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + _nested_filter_map_data.clear(); + + auto read_and_fill_data = [&](size_t before_rep_level_sz, size_t filter_map_index) { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + if (filter_map.has_filter()) { + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, before_rep_level_sz, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + } else { + RETURN_IF_ERROR(_nested_filter_map.init( + nullptr, _rep_levels.size() - before_rep_level_sz, false)); + } + + _null_run_lengths.clear(); + _ancestor_null_indices.clear(); + RETURN_IF_ERROR(gen_nested_null_map(before_rep_level_sz, _rep_levels.size(), + _null_run_lengths, _ancestor_null_indices)); + + { + SCOPED_RAW_TIMER(&_decode_null_map_time); + RETURN_IF_ERROR(_select_vector.init( + _null_run_lengths, + _rep_levels.size() - before_rep_level_sz - _ancestor_null_indices.size(), + map_data_column, &_nested_filter_map, 0, &_ancestor_null_indices)); + } + + DORIS_CHECK(_serde != nullptr); + RETURN_IF_ERROR(_chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector)); + if (!_ancestor_null_indices.empty()) { + RETURN_IF_ERROR(_chunk_reader->skip_values(_ancestor_null_indices.size(), false)); + } + if (filter_map.has_filter()) { + auto new_rep_sz = before_rep_level_sz; + for (size_t idx = before_rep_level_sz; idx < _rep_levels.size(); idx++) { + if (_nested_filter_map_data[idx - before_rep_level_sz]) { + _rep_levels[new_rep_sz] = _rep_levels[idx]; + _def_levels[new_rep_sz] = _def_levels[idx]; + new_rep_sz++; + } + } + _rep_levels.resize(new_rep_sz); + _def_levels.resize(new_rep_sz); + } + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + size_t right_row = std::min(left_row + batch_size - *read_rows, + (size_t)_row_ranges.get_range_to(_current_range_idx)); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + size_t load_rows = 0; + bool cross_page = false; + size_t before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &load_rows, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); + _filter_map_index += load_rows; + while (cross_page) { + before_rep_level_sz = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index - 1)); + } + *read_rows += load_rows; + _current_row_index += load_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_levels(FilterMap& filter_map, + size_t batch_size, + size_t* read_rows, + bool* eof) { + DORIS_CHECK(_in_nested); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + _rep_levels.clear(); + _def_levels.clear(); + *read_rows = 0; + + auto consume_level_segment = [&](size_t level_start, size_t filter_map_index) -> Status { + RETURN_IF_ERROR(_chunk_reader->fill_def(_def_levels)); + // Advance the encoded value stream without constructing a temporary Doris column. The + // definition levels identify the physical values that actually exist; dictionary skips + // still validate every index. + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(_def_levels, level_start)); + if (!filter_map.has_filter()) { + return Status::OK(); + } + + RETURN_IF_ERROR(gen_filter_map(filter_map, filter_map_index, level_start, + _rep_levels.size(), _nested_filter_map_data, + &_nested_filter_map)); + size_t write_index = level_start; + for (size_t read_index = level_start; read_index < _rep_levels.size(); ++read_index) { + if (_nested_filter_map_data[read_index - level_start] != 0) { + _rep_levels[write_index] = _rep_levels[read_index]; + _def_levels[write_index] = _def_levels[read_index]; + ++write_index; + } + } + _rep_levels.resize(write_index); + _def_levels.resize(write_index); + return Status::OK(); + }; + + while (_current_range_idx < _row_ranges.range_size()) { + const size_t left_row = + std::max(_current_row_index, _row_ranges.get_range_from(_current_range_idx)); + const size_t right_row = + std::min(left_row + batch_size - *read_rows, + static_cast(_row_ranges.get_range_to(_current_range_idx))); + _current_row_index = left_row; + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(left_row)); + + size_t loaded_rows = 0; + bool cross_page = false; + size_t level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, + &loaded_rows, &cross_page)); + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index)); + _filter_map_index += loaded_rows; + while (cross_page) { + level_start = _rep_levels.size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(_rep_levels, &cross_page)); + RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index - 1)); + } + + *read_rows += loaded_rows; + _current_row_index += loaded_rows; + _current_range_idx += (_current_row_index == _row_ranges.get_range_to(_current_range_idx)); + if (*read_rows == batch_size) { + break; + } + } + *eof = _current_range_idx == _row_ranges.range_size(); + return Status::OK(); +} + +template +Result +ScalarColumnReader::convert_dict_column_to_string_column( + const ColumnInt32* dict_column) { + DORIS_CHECK(dict_column != nullptr); + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + DORIS_CHECK(dictionary_decoder != nullptr); + const DataTypePtr dictionary_type = remove_nullable(_field_schema->data_type); + const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); + if (_materialization_state.dictionary_generation != + dictionary_decoder->dictionary_generation()) { + _materialization_state.typed_dictionary = dictionary_type->create_column(); + ParquetDecodeContext dictionary_context = _decode_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + dictionary_context.dictionary_index_only = false; + auto status = dictionary_serde->read_parquet_dictionary( + *_materialization_state.typed_dictionary, *dictionary_decoder, dictionary_context); + if (!status.ok()) { + return ResultError(std::move(status)); + } + DORIS_CHECK_EQ(_materialization_state.typed_dictionary->size(), + dictionary_decoder->dictionary_size()); + _materialization_state.dictionary_generation = dictionary_decoder->dictionary_generation(); + } + + auto result = _materialization_state.typed_dictionary->clone_empty(); + const auto& source_indices = dict_column->get_data(); + auto& indices = _materialization_state.dictionary_indices; + indices.resize(source_indices.size()); + for (size_t row = 0; row < source_indices.size(); ++row) { + if (UNLIKELY(source_indices[row] < 0 || + static_cast(source_indices[row]) >= + _materialization_state.typed_dictionary->size())) { + return ResultError(Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + source_indices[row], row, _materialization_state.typed_dictionary->size())); + } + indices[row] = static_cast(source_indices[row]); + } + result->insert_indices_from(*_materialization_state.typed_dictionary, indices.data(), + indices.data() + indices.size()); + return result; +} + +template +Status ScalarColumnReader::_try_load_dict_page(bool* loaded, + bool* has_dict) { + // _chunk_reader init will load first page header to check whether has dict page + *loaded = true; + *has_dict = _chunk_reader->has_dict(); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + const DataTypePtr target_type = remove_nullable(type); + const DataTypePtr source_type = remove_nullable(_field_schema->data_type); + const bool needs_logical_conversion = + !is_dict_filter && !serde_can_materialize_directly(source_type, target_type); + + ColumnPtr converted_source_column; + ColumnPtr* read_column = &doris_column; + DataTypePtr materialization_type = target_type; + if (needs_logical_conversion) { + if (_logical_converter == nullptr || _converter_source_type != source_type.get() || + _converter_target_type != target_type.get()) { + _logical_converter = converter::ColumnTypeConverter::get_converter( + source_type, target_type, converter::FileFormat::PARQUET); + if (!_logical_converter->support()) { + return Status::InternalError( + "The column type of '{}' has changed and is not supported: {}", + _field_schema->name, _logical_converter->get_error_msg()); + } + _converter_source_type = source_type.get(); + _converter_target_type = target_type.get(); + } + converted_source_column = _logical_converter->get_column(source_type, doris_column, type); + read_column = &converted_source_column; + materialization_type = remove_nullable(_logical_converter->get_type()); + } + + const DataTypePtr serde_type = + is_dict_filter ? remove_nullable(_field_schema->data_type) : materialization_type; + if (_serde_type != serde_type.get() || _dictionary_index_only != is_dict_filter) { + _serde_type = serde_type.get(); + _serde = serde_type->get_serde(); + _dictionary_index_only = is_dict_filter; + _materialization_state.reset_dictionary(); + } + _decode_context.dictionary_index_only = is_dict_filter; + + auto finish_logical_conversion = [&]() -> Status { + if (!needs_logical_conversion) { + return Status::OK(); + } + DORIS_CHECK(_logical_converter != nullptr); + doris_column = IColumn::mutate(std::move(doris_column)); + auto converted_column = doris_column->assert_mutable(); + if (is_column_nullable(*converted_column)) { + const auto* source_nullable = + check_and_get_column(*converted_source_column); + DORIS_CHECK(source_nullable != nullptr); + auto& destination_null_map = + assert_cast(*converted_column).get_null_map_data(); + const auto& source_null_map = source_nullable->get_null_map_data(); + destination_null_map.insert(source_null_map.begin(), source_null_map.end()); + } + SCOPED_RAW_TIMER(&_convert_time); + RETURN_IF_ERROR(_logical_converter->convert(converted_source_column, converted_column)); + doris_column = std::move(converted_column); + return Status::OK(); + }; + + _def_levels.clear(); + _rep_levels.clear(); + *read_rows = 0; + + if (_in_nested) { + RETURN_IF_ERROR(_read_nested_column(*read_column, materialization_type, filter_map, + batch_size, read_rows, eof, is_dict_filter)); + return finish_logical_conversion(); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + + do { + // generate the row ranges that should be read + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + // skip the whole page + _current_row_index = right_row; + } else { + bool skip_whole_batch = false; + // Determining whether to skip page or batch will increase the calculation time. + // When the filtering effect is greater than 60%, it is possible to skip the page or batch. + if (filter_map.has_filter() && filter_map.filter_ratio() > 0.6) { + // lazy read + size_t remaining_num_values = read_ranges.count(); + if (batch_size >= remaining_num_values && + filter_map.can_filter_all(remaining_num_values, _filter_map_index)) { + // We can skip the whole page if the remaining values are filtered by predicate columns + _filter_map_index += remaining_num_values; + _current_row_index = right_row; + *read_rows = remaining_num_values; + break; + } + skip_whole_batch = batch_size <= remaining_num_values && + filter_map.can_filter_all(batch_size, _filter_map_index); + if (skip_whole_batch) { + _filter_map_index += batch_size; + } + } + // load page data to decode or skip values + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); idx++) { + auto range = read_ranges.get_range(idx); + // generate the skipped values + size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + // generate the read values + size_t read_values = + std::min((size_t)(range.to() - range.from()), batch_size - has_read); + if (skip_whole_batch) { + RETURN_IF_ERROR(_skip_values(read_values)); + } else { + RETURN_IF_ERROR(_read_values(read_values, *read_column, materialization_type, + filter_map, is_dict_filter)); + } + has_read += read_values; + *read_rows += read_values; + _current_row_index += read_values; + if (has_read == batch_size) { + break; + } + } + } + } while (false); + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + + return finish_logical_conversion(); +} + +Status ArrayColumnReader::init(std::unique_ptr element_reader, FieldSchema* field) { + _field_schema = field; + _element_reader = std::move(element_reader); + return Status::OK(); +} + +Status ArrayColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_ARRAY) { + return Status::Corruption( + "Wrong data type for column '{}', expected Array type, actual type: {}.", + _field_schema->name, type->get_name()); + } + + ColumnPtr& element_column = assert_cast(*data_column).get_data_ptr(); + const DataTypePtr& element_type = + (assert_cast(remove_nullable(type).get()))->get_nested_type(); + // read nested column + RETURN_IF_ERROR(_element_reader->read_column_data(element_column, element_type, + root_node->get_element_node(), filter_map, + batch_size, read_rows, eof, is_dict_filter)); + if (*read_rows == 0) { + return Status::OK(); + } + + ColumnArray::Offsets64& offsets_data = assert_cast(*data_column).get_offsets(); + // fill offset and null map + fill_array_offset(_field_schema, offsets_data, null_map_ptr, _element_reader->get_rep_level(), + _element_reader->get_def_level()); + DCHECK_EQ(element_column->size(), offsets_data.back()); +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::init(std::unique_ptr key_reader, + std::unique_ptr value_reader, FieldSchema* field) { + _field_schema = field; + _key_reader = std::move(key_reader); + _value_reader = std::move(value_reader); + return Status::OK(); +} + +Status MapColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (remove_nullable(type)->get_primitive_type() != PrimitiveType::TYPE_MAP) { + return Status::Corruption( + "Wrong data type for column '{}', expected Map type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& map = assert_cast(*data_column); + const DataTypePtr& key_type = + assert_cast(remove_nullable(type).get())->get_key_type(); + const DataTypePtr& value_type = + assert_cast(remove_nullable(type).get())->get_value_type(); + ColumnPtr& key_column = map.get_keys_ptr(); + ColumnPtr& value_column = map.get_values_ptr(); + + size_t key_rows = 0; + size_t value_rows = 0; + bool key_eof = false; + bool value_eof = false; + int64_t orig_col_column_size = key_column->size(); + + RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->get_key_node(), + filter_map, batch_size, &key_rows, &key_eof, + is_dict_filter)); + + while (value_rows < key_rows && !value_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_value_reader->read_column_data( + value_column, value_type, root_node->get_value_node(), filter_map, + key_rows - value_rows, &loop_rows, &value_eof, is_dict_filter, + key_column->size() - orig_col_column_size)); + value_rows += loop_rows; + } + DCHECK_EQ(key_rows, value_rows); + *read_rows = key_rows; + *eof = key_eof; + + if (*read_rows == 0) { + return Status::OK(); + } + + DCHECK_EQ(key_column->size(), value_column->size()); + // fill offset and null map + fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, _key_reader->get_rep_level(), + _key_reader->get_def_level()); + DCHECK_EQ(key_column->size(), map.get_offsets().back()); +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +Status MapColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + DORIS_CHECK(dynamic_cast(_key_reader.get()) == nullptr); + return _key_reader->read_column_levels(filter_map, batch_size, read_rows, eof); +} + +Status StructColumnReader::init( + std::unordered_map>&& child_readers, + FieldSchema* field) { + _field_schema = field; + _child_readers = std::move(child_readers); + return Status::OK(); +} + +Status StructColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof) { + _read_column_names.clear(); + for (const auto& child : _field_schema->children) { + auto reader = _child_readers.find(child.name); + DORIS_CHECK(reader != _child_readers.end()); + if (dynamic_cast(reader->second.get()) != nullptr) { + continue; + } + _read_column_names.emplace_back(child.name); + return reader->second->read_column_levels(filter_map, batch_size, read_rows, eof); + } + return Status::InternalError("Struct {} has no physical reader for levels", + _field_schema->name); +} +Status StructColumnReader::read_column_data( + ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, FilterMap& filter_map, + size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { + MutableColumnPtr data_column; + NullMap* null_map_ptr = nullptr; + doris_column = IColumn::mutate(std::move(doris_column)); + if (is_column_nullable(*doris_column)) { + auto mutable_column = doris_column->assert_mutable(); + auto* nullable_column = assert_cast(mutable_column.get()); + null_map_ptr = &nullable_column->get_null_map_data(); + data_column = nullable_column->get_nested_column_ptr(); + } else { + if (_field_schema->data_type->is_nullable()) { + return Status::Corruption("Not nullable column has null values in parquet file"); + } + data_column = doris_column->assert_mutable(); + } + if (type->get_primitive_type() != PrimitiveType::TYPE_STRUCT) { + return Status::Corruption( + "Wrong data type for column '{}', expected Struct type, actual type id {}.", + _field_schema->name, type->get_name()); + } + + auto& doris_struct = assert_cast(*data_column); + const auto* doris_struct_type = assert_cast(remove_nullable(type).get()); + + int64_t not_missing_column_id = -1; + size_t not_missing_orig_column_size = 0; + std::vector missing_column_idxs {}; + std::vector skip_reading_column_idxs {}; + + _read_column_names.clear(); + + for (size_t i = 0; i < doris_struct.tuple_size(); ++i) { + ColumnPtr& doris_field = doris_struct.get_column_ptr(i); + auto& doris_type = doris_struct_type->get_element(i); + auto& doris_name = doris_struct_type->get_element_name(i); + if (!root_node->children_column_exists(doris_name)) { + missing_column_idxs.push_back(i); + VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i + << "], doris_name: " << doris_name << " (column not exists in root node)"; + continue; + } + auto file_name = root_node->children_file_column_name(doris_name); + + // Check if this is a SkipReadingReader - we should skip it when choosing reference column + // because SkipReadingReader doesn't know the actual data size in nested context + bool is_skip_reader = + dynamic_cast(_child_readers[file_name].get()) != nullptr; + + if (is_skip_reader) { + // Store SkipReadingReader columns to fill them later based on reference column size + skip_reading_column_idxs.push_back(i); + continue; + } + + // Only add non-SkipReadingReader columns to _read_column_names + // This ensures get_rep_level() and get_def_level() return valid levels + _read_column_names.emplace_back(file_name); + + size_t field_rows = 0; + bool field_eof = false; + if (not_missing_column_id == -1) { + not_missing_column_id = i; + not_missing_orig_column_size = doris_field->size(); + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->get_children_node(doris_name), filter_map, + batch_size, &field_rows, &field_eof, is_dict_filter)); + *read_rows = field_rows; + *eof = field_eof; + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column + * (since `_read_nested_column` is not called repeatedly). + * + * It is worth mentioning that, theoretically, any sub-column can be chosen to fill the null_map, + * and selecting the shortest one will offer better performance + */ + } else { + while (field_rows < *read_rows && !field_eof) { + size_t loop_rows = 0; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->get_children_node(doris_name), + filter_map, *read_rows - field_rows, &loop_rows, &field_eof, + is_dict_filter)); + field_rows += loop_rows; + } + DCHECK_EQ(*read_rows, field_rows); + // DCHECK_EQ(*eof, field_eof); + } + } + + int64_t missing_column_sz = -1; + + if (not_missing_column_id == -1) { + // All queried columns are missing in the file (e.g., all added after schema change) + // We need to pick a column from _field_schema children that exists in the file for RL/DL reference + std::string reference_file_column_name; + std::unique_ptr* reference_reader = nullptr; + + for (const auto& child : _field_schema->children) { + auto it = _child_readers.find(child.name); + if (it != _child_readers.end()) { + // Skip SkipReadingReader as they don't have valid RL/DL + bool is_skip_reader = dynamic_cast(it->second.get()) != nullptr; + if (!is_skip_reader) { + reference_file_column_name = child.name; + reference_reader = &(it->second); + break; + } + } + } + + if (reference_reader != nullptr) { + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR( + (*reference_reader) + ->read_column_levels(filter_map, batch_size, &field_rows, &field_eof)); + + *read_rows = field_rows; + *eof = field_eof; + _read_column_names.emplace_back(reference_file_column_name); + missing_column_sz = 0; + const auto& rep_levels = (*reference_reader)->get_rep_level(); + const auto& def_levels = (*reference_reader)->get_def_level(); + DORIS_CHECK_EQ(rep_levels.size(), def_levels.size()); + for (size_t level_index = 0; level_index < def_levels.size(); ++level_index) { + if (def_levels[level_index] >= _field_schema->repeated_parent_def_level && + rep_levels[level_index] <= _field_schema->repetition_level) { + ++missing_column_sz; + } + } + } else { + return Status::Corruption( + "Cannot read struct '{}': all queried columns are missing and no reference " + "column found in file", + _field_schema->name); + } + } + + // This missing_column_sz is not *read_rows. Because read_rows returns the number of rows. + // For example: suppose we have a column array>, + // where b is a newly added column, that is, a missing column. + // There are two rows of data in this column, + // [{1,null},{2,null},{3,null}] + // [{4,null},{5,null}] + // When you first read subcolumn a, you read 5 data items and the value of *read_rows is 2. + // You should insert 5 records into subcolumn b instead of 2. + if (missing_column_sz == -1) { + missing_column_sz = doris_struct.get_column(not_missing_column_id).size() - + not_missing_orig_column_size; + } + + // Fill SkipReadingReader columns with the correct amount of data based on reference column + // Let SkipReadingReader handle the data filling through its read_column_data method + for (auto idx : skip_reading_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = const_cast(doris_struct_type->get_element(idx)); + auto& doris_name = const_cast(doris_struct_type->get_element_name(idx)); + auto file_name = root_node->children_file_column_name(doris_name); + + size_t field_rows = 0; + bool field_eof = false; + RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( + doris_field, doris_type, root_node->get_children_node(doris_name), filter_map, + missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz)); + } + + // Fill truly missing columns (not in root_node) with null or default value + for (auto idx : missing_column_idxs) { + auto& doris_field = doris_struct.get_column_ptr(idx); + auto& doris_type = doris_struct_type->get_element(idx); + DCHECK(doris_type->is_nullable()); + doris_field = IColumn::mutate(std::move(doris_field)); + auto mutable_column = doris_field->assert_mutable(); + auto* nullable_column = static_cast(mutable_column.get()); + nullable_column->insert_many_defaults(missing_column_sz); + } + + if (null_map_ptr != nullptr) { + fill_struct_null_map(_field_schema, *null_map_ptr, this->get_rep_level(), + this->get_def_level()); + } +#ifndef NDEBUG + doris_column->sanity_check(); +#endif + return Status::OK(); +} + +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; +template class ScalarColumnReader; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h new file mode 100644 index 00000000000000..847d9d35204077 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -0,0 +1,570 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "format/column_type_convert.h" +#include "format/generic_reader.h" +#include "format/parquet/parquet_common.h" +#include "format/table/table_schema_change_helper.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace cctz { +class time_zone; +} // namespace cctz + +namespace doris::io { +struct IOContext; +} // namespace doris::io + +namespace doris::format::parquet::native { +using ::doris::FieldSchema; +using ::doris::ColumnString; + +class ColumnReader { +public: + struct ColumnStatistics { + ColumnStatistics() + : page_index_read_calls(0), + decompress_time(0), + decompress_cnt(0), + decode_header_time(0), + decode_value_time(0), + materialization_time(0), + decode_dict_time(0), + decode_level_time(0), + decode_null_map_time(0), + convert_time(0), + skip_page_header_num(0), + parse_page_header_num(0), + read_page_header_time(0), + page_read_counter(0), + page_cache_write_counter(0), + page_cache_compressed_write_counter(0), + page_cache_decompressed_write_counter(0), + page_cache_hit_counter(0), + page_cache_missing_counter(0), + page_cache_compressed_hit_counter(0), + page_cache_decompressed_hit_counter(0) {} + + ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time, + int64_t convert_time_) + : page_index_read_calls(0), + decompress_time(cs.decompress_time), + decompress_cnt(cs.decompress_cnt), + decode_header_time(cs.decode_header_time), + decode_value_time(cs.decode_value_time), + materialization_time(cs.materialization_time), + decode_dict_time(cs.decode_dict_time), + decode_level_time(cs.decode_level_time), + decode_null_map_time(null_map_time), + convert_time(convert_time_), + skip_page_header_num(cs.skip_page_header_num), + parse_page_header_num(cs.parse_page_header_num), + read_page_header_time(cs.read_page_header_time), + page_read_counter(cs.page_read_counter), + page_cache_write_counter(cs.page_cache_write_counter), + page_cache_compressed_write_counter(cs.page_cache_compressed_write_counter), + page_cache_decompressed_write_counter(cs.page_cache_decompressed_write_counter), + page_cache_hit_counter(cs.page_cache_hit_counter), + page_cache_missing_counter(cs.page_cache_missing_counter), + page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter), + page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter) {} + + int64_t page_index_read_calls; + int64_t decompress_time; + int64_t decompress_cnt; + int64_t decode_header_time; + int64_t decode_value_time; + int64_t materialization_time; + int64_t decode_dict_time; + int64_t decode_level_time; + int64_t decode_null_map_time; + int64_t convert_time; + int64_t skip_page_header_num; + int64_t parse_page_header_num; + int64_t read_page_header_time; + int64_t page_read_counter; + int64_t page_cache_write_counter; + int64_t page_cache_compressed_write_counter; + int64_t page_cache_decompressed_write_counter; + int64_t page_cache_hit_counter; + int64_t page_cache_missing_counter; + int64_t page_cache_compressed_hit_counter; + int64_t page_cache_decompressed_hit_counter; + + void merge(ColumnStatistics& col_statistics) { + page_index_read_calls += col_statistics.page_index_read_calls; + decompress_time += col_statistics.decompress_time; + decompress_cnt += col_statistics.decompress_cnt; + decode_header_time += col_statistics.decode_header_time; + decode_value_time += col_statistics.decode_value_time; + materialization_time += col_statistics.materialization_time; + decode_dict_time += col_statistics.decode_dict_time; + decode_level_time += col_statistics.decode_level_time; + decode_null_map_time += col_statistics.decode_null_map_time; + convert_time += col_statistics.convert_time; + skip_page_header_num += col_statistics.skip_page_header_num; + parse_page_header_num += col_statistics.parse_page_header_num; + read_page_header_time += col_statistics.read_page_header_time; + page_read_counter += col_statistics.page_read_counter; + page_cache_write_counter += col_statistics.page_cache_write_counter; + page_cache_compressed_write_counter += + col_statistics.page_cache_compressed_write_counter; + page_cache_decompressed_write_counter += + col_statistics.page_cache_decompressed_write_counter; + page_cache_hit_counter += col_statistics.page_cache_hit_counter; + page_cache_missing_counter += col_statistics.page_cache_missing_counter; + page_cache_compressed_hit_counter += col_statistics.page_cache_compressed_hit_counter; + page_cache_decompressed_hit_counter += + col_statistics.page_cache_decompressed_hit_counter; + } + }; + + ColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} + virtual ~ColumnReader() = default; + virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter, + int64_t real_column_size = -1) = 0; + + // Consume a nested batch while retaining only definition/repetition levels. This is used when + // schema evolution makes every projected STRUCT child synthetic: the parent still needs one + // physical leaf's shape, but decoding that leaf's strings or other payload would be wasted. + virtual Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) = 0; + + virtual Result convert_dict_column_to_string_column( + const ColumnInt32* dict_column) { + throw Exception( + Status::FatalError("Method convert_dict_column_to_string_column is not supported")); + } + + static Status create(io::FileReaderSPtr file, FieldSchema* field, + const tparquet::RowGroup& row_group, const RowRanges& row_ranges, + const cctz::time_zone* ctz, io::IOContext* io_ctx, + std::unique_ptr& reader, size_t max_buf_size, + std::unordered_map& col_offsets, + RuntimeState* state, bool in_collection = false, + const std::set& column_ids = {}, + const std::set& filter_column_ids = {}); + virtual const std::vector& get_rep_level() const = 0; + virtual const std::vector& get_def_level() const = 0; + virtual ColumnStatistics column_statistics() = 0; + virtual void close() = 0; + + virtual void reset_filter_map_index() = 0; + + FieldSchema* get_field_schema() const { return _field_schema; } + void set_column_in_nested() { _in_nested = true; } + +protected: + void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const; + + FieldSchema* _field_schema = nullptr; + const RowRanges& _row_ranges; + size_t _total_rows = 0; + const cctz::time_zone* _ctz = nullptr; + io::IOContext* _io_ctx = nullptr; + int64_t _current_row_index = 0; + int64_t _decode_null_map_time = 0; + + size_t _filter_map_index = 0; + std::set _filter_column_ids; + + // _in_nested: column in struct/map/array + // IN_COLLECTION : column in map/array + bool _in_nested = false; +}; + +template +class ScalarColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ScalarColumnReader) +public: + ScalarColumnReader(const RowRanges& row_ranges, size_t total_rows, + const tparquet::ColumnChunk& chunk_meta, + const tparquet::OffsetIndex* offset_index, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx), + _chunk_meta(chunk_meta), + _offset_index(offset_index) {} + ~ScalarColumnReader() override { close(); } + Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, + RuntimeState* state); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + Result convert_dict_column_to_string_column( + const ColumnInt32* dict_column) override; + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { + return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time, + _convert_time); + } + void close() override {} + + void reset_filter_map_index() override { + _filter_map_index = 0; // nested + } + +private: + tparquet::ColumnChunk _chunk_meta; + const tparquet::OffsetIndex* _offset_index = nullptr; + std::unique_ptr _stream_reader; + std::unique_ptr> _chunk_reader; + // rep def levels buffer. + std::vector _rep_levels; + std::vector _def_levels; + + size_t _current_range_idx = 0; + + Status gen_nested_null_map(size_t level_start_idx, size_t level_end_idx, + std::vector& null_map, + std::unordered_set& ancestor_null_indices) { + size_t has_read = level_start_idx; + null_map.emplace_back(0); + bool prev_is_null = false; + + while (has_read < level_end_idx) { + level_t def_level = _def_levels[has_read++]; + size_t loop_read = 1; + while (has_read < _def_levels.size() && _def_levels[has_read] == def_level) { + has_read++; + loop_read++; + } + + if (def_level < _field_schema->repeated_parent_def_level) { + for (size_t i = 0; i < loop_read; i++) { + ancestor_null_indices.insert(has_read - level_start_idx - loop_read + i); + } + continue; + } + + bool is_null = def_level < _field_schema->definition_level; + + if (prev_is_null == is_null && (USHRT_MAX - null_map.back() >= loop_read)) { + null_map.back() += loop_read; + } else { + if (!(prev_is_null ^ is_null)) { + null_map.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + null_map.emplace_back(USHRT_MAX); + null_map.emplace_back(0); + remaining -= USHRT_MAX; + } + null_map.emplace_back((u_short)remaining); + prev_is_null = is_null; + } + } + return Status::OK(); + } + + Status gen_filter_map(FilterMap& filter_map, size_t filter_loc, size_t level_start_idx, + size_t level_end_idx, std::vector& nested_filter_map_data, + FilterMap* nested_filter_map) { + DORIS_CHECK(nested_filter_map != nullptr); + nested_filter_map_data.resize(level_end_idx - level_start_idx); + for (size_t idx = level_start_idx; idx < level_end_idx; idx++) { + if (idx != level_start_idx && _rep_levels[idx] == 0) { + filter_loc++; + } + nested_filter_map_data[idx - level_start_idx] = + filter_map.filter_map_data()[filter_loc]; + } + + return nested_filter_map->init(nested_filter_map_data.data(), nested_filter_map_data.size(), + false); + } + + DataTypeSerDeSPtr _serde; + const IDataType* _serde_type = nullptr; + std::unique_ptr _logical_converter; + const IDataType* _converter_source_type = nullptr; + const IDataType* _converter_target_type = nullptr; + ParquetDecodeContext _decode_context; + ParquetMaterializationState _materialization_state; + bool _dictionary_index_only = false; + // Batch scratch is retained by the persistent leaf reader. Only logical sizes are reset, so + // adaptive batch-size changes and repeated complex reads do not allocate fresh level plans. + std::vector _null_run_lengths; + std::unordered_set _ancestor_null_indices; + std::vector _nested_filter_map_data; + FilterMap _nested_filter_map; + ColumnSelectVector _select_vector; + int64_t _convert_time = 0; + + Status _skip_values(size_t num_values); + Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, bool is_dict_filter); + Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof, bool is_dict_filter); + Status _try_load_dict_page(bool* loaded, bool* has_dict); +}; + +class ArrayColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(ArrayColumnReader) +public: + ArrayColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~ArrayColumnReader() override { close(); } + Status init(std::unique_ptr element_reader, FieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override { + return _element_reader->read_column_levels(filter_map, batch_size, read_rows, eof); + } + const std::vector& get_rep_level() const override { + return _element_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _element_reader->get_def_level(); + } + ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); } + void close() override {} + + void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); } + +private: + std::unique_ptr _element_reader; +}; + +class MapColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(MapColumnReader) +public: + MapColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~MapColumnReader() override { close(); } + + Status init(std::unique_ptr key_reader, + std::unique_ptr value_reader, FieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + return _key_reader->get_rep_level(); + } + const std::vector& get_def_level() const override { + return _key_reader->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics kst = _key_reader->column_statistics(); + ColumnStatistics vst = _value_reader->column_statistics(); + kst.merge(vst); + return kst; + } + + void close() override {} + + void reset_filter_map_index() override { + _key_reader->reset_filter_map_index(); + _value_reader->reset_filter_map_index(); + } + +private: + std::unique_ptr _key_reader; + std::unique_ptr _value_reader; +}; + +class StructColumnReader : public ColumnReader { + ENABLE_FACTORY_CREATOR(StructColumnReader) +public: + StructColumnReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} + ~StructColumnReader() override { close(); } + + Status init(std::unordered_map>&& child_readers, + FieldSchema* field); + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, + bool* eof) override; + + const std::vector& get_rep_level() const override { + if (!_read_column_names.empty()) { + // can't use _child_readers[*_read_column_names.begin()] + // because the operator[] of std::unordered_map is not const :( + /* + * Considering the issue in the `_read_nested_column` function where data may span across pages, leading + * to missing definition and repetition levels, when filling the null_map of the struct later, it is + * crucial to use the definition and repetition levels from the first read column, + * that is `_read_column_names.front()`. + */ + return _child_readers.find(_read_column_names.front())->second->get_rep_level(); + } + return _child_readers.begin()->second->get_rep_level(); + } + + const std::vector& get_def_level() const override { + if (!_read_column_names.empty()) { + return _child_readers.find(_read_column_names.front())->second->get_def_level(); + } + return _child_readers.begin()->second->get_def_level(); + } + + ColumnStatistics column_statistics() override { + ColumnStatistics st; + for (const auto& column_name : _read_column_names) { + auto reader = _child_readers.find(column_name); + if (reader != _child_readers.end()) { + ColumnStatistics cst = reader->second->column_statistics(); + st.merge(cst); + } + } + return st; + } + + void close() override {} + + void reset_filter_map_index() override { + for (const auto& reader : _child_readers) { + reader.second->reset_filter_map_index(); + } + } + +private: + std::unordered_map> _child_readers; + std::vector _read_column_names; + //Need to use vector instead of set,see `get_rep_level()` for the reason. +}; + +// A special reader that skips actual reading but provides empty data with correct structure +// This is used when a column is not needed but its structure is required (e.g., for map keys) +class SkipReadingReader : public ColumnReader { +public: + SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, + io::IOContext* io_ctx, FieldSchema* field_schema) + : ColumnReader(row_ranges, total_rows, ctz, io_ctx) { + _field_schema = field_schema; // Use inherited member from base class + VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: " + << _field_schema->name; + } + + Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, + bool is_dict_filter, int64_t real_column_size = -1) override { + VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: " + << _field_schema->name << ", batch_size: " << batch_size; + DCHECK(real_column_size >= 0); // real_column_size for filtered column size. + + // Simulate reading without actually reading data + // Fill with default/null values based on column type + doris_column = IColumn::mutate(std::move(doris_column)); + MutableColumnPtr data_column = doris_column->assert_mutable(); + + if (real_column_size > 0) { + if (is_column_nullable(*doris_column)) { + auto* nullable_column = static_cast(data_column.get()); + nullable_column->insert_many_defaults(real_column_size); + } else { + // For non-nullable columns, insert appropriate default values + for (size_t i = 0; i < real_column_size; ++i) { + data_column->insert_default(); + } + } + } + + *read_rows = batch_size; // Indicate we "read" batch_size rows + *eof = false; // We can always provide more empty data + + VLOG_DEBUG << "[ParquetReader] SkipReadingReader generated " << batch_size + << " default values for field: " << _field_schema->name; + + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t*, bool*) override { + return Status::InternalError("Skip reader cannot provide Parquet levels for field {}", + _field_schema->name); + } + + static std::unique_ptr create_unique(const RowRanges& row_ranges, + size_t total_rows, cctz::time_zone* ctz, + io::IOContext* io_ctx, + FieldSchema* field_schema) { + return std::make_unique(row_ranges, total_rows, ctz, io_ctx, + field_schema); + } + + // These methods should not be called for SkipReadingReader + // If they are called, it indicates a logic error in the code + const std::vector& get_rep_level() const override { + LOG(FATAL) << "get_rep_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + const std::vector& get_def_level() const override { + LOG(FATAL) << "get_def_level() should not be called on SkipReadingReader for field: " + << _field_schema->name + << ". This indicates the SkipReadingReader was incorrectly used as a reference " + "column."; + __builtin_unreachable(); + } + + // Implement required pure virtual methods from base class + ColumnStatistics column_statistics() override { + return ColumnStatistics(); // Return empty statistics + } + + void close() override { + // Nothing to close for skip reading + } + + void reset_filter_map_index() override { _filter_map_index = 0; } +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp b/be/src/format_v2/parquet/reader/native/decoder.cpp new file mode 100644 index 00000000000000..c42f09af75d18e --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.cpp @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/decoder.h" + +#include +#include + +#include "format_v2/parquet/reader/native/bool_plain_decoder.h" +#include "format_v2/parquet/reader/native/bool_rle_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" +#include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" +#include "format_v2/parquet/reader/native/fix_length_dict_decoder.hpp" +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +namespace doris::format::parquet::native { +Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder) { + switch (encoding) { + case tparquet::Encoding::PLAIN: + switch (type) { + case tparquet::Type::BOOLEAN: + decoder.reset(new BoolPlainDecoder()); + break; + case tparquet::Type::BYTE_ARRAY: + decoder.reset(new ByteArrayPlainDecoder()); + break; + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder.reset(new FixLengthPlainDecoder()); + break; + default: + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); + } + break; + case tparquet::Encoding::RLE_DICTIONARY: + switch (type) { + case tparquet::Type::BOOLEAN: + return Status::InternalError("Bool type can't has dictionary page"); + case tparquet::Type::BYTE_ARRAY: + decoder.reset(new ByteArrayDictDecoder()); + break; + case tparquet::Type::INT32: + decoder.reset(new FixLengthDictDecoder()); + break; + case tparquet::Type::INT64: + decoder.reset(new FixLengthDictDecoder()); + break; + case tparquet::Type::INT96: + decoder.reset(new FixLengthDictDecoder()); + break; + case tparquet::Type::FLOAT: + decoder.reset(new FixLengthDictDecoder()); + break; + case tparquet::Type::DOUBLE: + decoder.reset(new FixLengthDictDecoder()); + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder.reset(new FixLengthDictDecoder()); + break; + default: + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); + } + break; + case tparquet::Encoding::RLE: + switch (type) { + case tparquet::Type::BOOLEAN: + decoder.reset(new BoolRLEDecoder()); + break; + default: + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); + } + break; + case tparquet::Encoding::DELTA_BINARY_PACKED: + // Supports only INT32 and INT64. + switch (type) { + case tparquet::Type::INT32: + decoder.reset(new DeltaBitPackDecoder()); + break; + case tparquet::Type::INT64: + decoder.reset(new DeltaBitPackDecoder()); + break; + default: + return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); + } + break; + case tparquet::Encoding::DELTA_BYTE_ARRAY: + switch (type) { + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder.reset(new DeltaByteArrayDecoder()); + break; + default: + return Status::InternalError( + "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); + } + break; + case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: + switch (type) { + case tparquet::Type::BYTE_ARRAY: + decoder.reset(new DeltaLengthByteArrayDecoder()); + break; + default: + return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); + } + break; + case tparquet::Encoding::BYTE_STREAM_SPLIT: + switch (type) { + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder.reset(new ByteStreamSplitDecoder()); + break; + default: + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); + } + break; + default: + return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", + tparquet::to_string(encoding), tparquet::to_string(type)); + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native \ No newline at end of file diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h new file mode 100644 index 00000000000000..ab48e0f34b1330 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -0,0 +1,143 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/custom_allocator.h" +#include "core/data_type_serde/parquet_decode_source.h" +#include "core/types.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class Decoder : public ParquetDecodeSource { +public: + Decoder() = default; + virtual ~Decoder() = default; + + static Status get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, + std::unique_ptr& decoder); + + // The type with fix length + void set_type_length(int32_t type_length) { _type_length = type_length; } + + // Set the data to be decoded + virtual Status set_data(Slice* data) { + _data = data; + _offset = 0; + return Status::OK(); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + return Status::NotSupported("Fixed values are not supported by this Parquet decoder"); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + return Status::NotSupported("Binary values are not supported by this Parquet decoder"); + } + + Status skip_values(size_t num_values) override = 0; + + virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) { + return Status::NotSupported("set_dict is not supported"); + } + +protected: + int32_t _type_length; + Slice* _data = nullptr; + uint32_t _offset = 0; +}; + +class BaseDictDecoder : public Decoder { +public: + BaseDictDecoder() = default; + ~BaseDictDecoder() override = default; + + // Set the data to be decoded + Status set_data(Slice* data) override { + if (UNLIKELY(data == nullptr || data->size == 0)) { + return Status::Corruption("Parquet dictionary index stream is empty"); + } + _data = data; + _offset = 0; + uint8_t bit_width = *data->data; + _index_batch_decoder = std::make_unique>( + reinterpret_cast(data->data) + 1, static_cast(data->size) - 1, + bit_width); + return Status::OK(); + } + + bool has_dictionary() const override { return true; } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + indices->resize(num_values); + const auto decoded = + _index_batch_decoder->GetBatch(indices->data(), cast_set(num_values)); + if (UNLIKELY(decoded != num_values)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY((*indices)[row] >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[row], row, num_dictionary_values); + } + } + return Status::OK(); + } + +protected: + Status skip_values(size_t num_values) override { + _skip_indices.resize(num_values); + const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), + cast_set(num_values)); + if (UNLIKELY(skipped != num_values)) { + return Status::IOError("Can't skip enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(_skip_indices[row] >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at skipped row {} exceeds dictionary size {}", + _skip_indices[row], row, num_dictionary_values); + } + } + return Status::OK(); + } + + // For dictionary encoding + DorisUniqueBufferPtr _dict; + std::unique_ptr> _index_batch_decoder; + std::vector _skip_indices; + uint64_t _dictionary_generation = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp new file mode 100644 index 00000000000000..dac29ccfc5d4d4 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" + +namespace doris::format::parquet::native { +Status DeltaLengthByteArrayDecoder::_decode_lengths() { + RETURN_IF_ERROR(_len_decoder.set_bit_reader(_bit_reader)); + // get the number of encoded lengths + int num_length = _len_decoder.valid_values_count(); + _buffered_length.resize(num_length); + + // decode all the lengths. all the lengths are buffered in buffered_length_. + uint32_t ret; + RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), num_length, &ret)); + DCHECK_EQ(ret, num_length); + _length_idx = 0; + _num_valid_values = num_length; + return Status::OK(); +} + +Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, + int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + + int32_t data_size = 0; + const int32_t* length_ptr = _buffered_length.data() + _length_idx; + for (int i = 0; i < max_values; ++i) { + int32_t len = length_ptr[i]; + if (len < 0) [[unlikely]] { + return Status::InvalidArgument("Negative string delta length"); + } + buffer[i].size = len; + if (common::add_overflow(data_size, len, data_size)) { + return Status::InvalidArgument("Excess expansion in DELTA_(LENGTH_)BYTE_ARRAY"); + } + } + _length_idx += max_values; + + _buffered_data.resize(data_size); + char* data_ptr = _buffered_data.data(); + for (int j = 0; j < data_size; j++) { + if (!_bit_reader->GetValue(8, data_ptr + j)) { + return Status::IOError("Get length bytes EOF"); + } + } + + for (int i = 0; i < max_values; ++i) { + buffer[i].data = data_ptr; + data_ptr += buffer[i].size; + } + // this->num_values_ -= max_values; + _num_valid_values -= max_values; + *out_num_values = max_values; + return Status::OK(); +} + +Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* out_num_values) { + // Decode up to `max_values` strings into an internal buffer + // and reference them into `buffer`. + max_values = std::min(max_values, _num_valid_values); + if (max_values == 0) { + *out_num_values = max_values; + return Status::OK(); + } + + int suffix_read; + RETURN_IF_ERROR(_suffix_decoder.decode(buffer, max_values, &suffix_read)); + if (suffix_read != max_values) [[unlikely]] { + return Status::IOError("Read {}, expecting {} from suffix decoder", + std::to_string(suffix_read), std::to_string(max_values)); + } + + int64_t data_size = 0; + const int32_t* prefix_len_ptr = _buffered_prefix_length.data() + _prefix_len_offset; + for (int i = 0; i < max_values; ++i) { + if (prefix_len_ptr[i] < 0) [[unlikely]] { + return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); + } + if (common::add_overflow(data_size, static_cast(prefix_len_ptr[i]), data_size) || + common::add_overflow(data_size, static_cast(buffer[i].size), data_size)) + [[unlikely]] { + return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); + } + } + _buffered_data.resize(data_size); + + std::string_view prefix {_last_value}; + + char* data_ptr = _buffered_data.data(); + for (int i = 0; i < max_values; ++i) { + if (static_cast(prefix_len_ptr[i]) > prefix.length()) [[unlikely]] { + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + memcpy(data_ptr, prefix.data(), prefix_len_ptr[i]); + // buffer[i] currently points to the string suffix + memcpy(data_ptr + prefix_len_ptr[i], buffer[i].data, buffer[i].size); + buffer[i].data = data_ptr; + buffer[i].size += prefix_len_ptr[i]; + data_ptr += buffer[i].size; + prefix = std::string_view {buffer[i].data, buffer[i].size}; + } + _prefix_len_offset += max_values; + _num_valid_values -= max_values; + _last_value = std::string {prefix}; + + if (_num_valid_values == 0) { + _last_value_in_previous_page = _last_value; + } + *out_num_values = max_values; + return Status::OK(); +} +} // namespace doris::format::parquet::native \ No newline at end of file diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h new file mode 100644 index 00000000000000..4358d3908114be --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -0,0 +1,438 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type.h" +#include "exec/common/arithmetic_overflow.h" +#include "format/parquet/parquet_common.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/bit_stream_utils.h" +#include "util/bit_stream_utils.inline.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class DeltaDecoder : public Decoder { +public: + DeltaDecoder() = default; + ~DeltaDecoder() override = default; +}; + +/** + * Format + * [header] [block 1] [block 2] ... [block N] + * Header + * [block size] [_mini_blocks_per_block] [_total_value_count] [first value] + * Block + * [min delta] [list of bitwidths of the mini blocks] [miniblocks] + */ +template +class DeltaBitPackDecoder final : public DeltaDecoder { +public: + using UT = std::make_unsigned_t; + + DeltaBitPackDecoder() = default; + ~DeltaBitPackDecoder() override = default; + + Status skip_values(size_t num_values) override { + _values.resize(num_values); + uint32_t num_valid_values; + return _get_internal(_values.data(), cast_set(num_values), &num_valid_values); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + _values.resize(num_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", num_values, + decoded_count); + } + return consumer.consume(reinterpret_cast(_values.data()), num_values, + sizeof(T)); + } + + Status decode(T* buffer, uint32_t num_values, uint32_t* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + uint32_t valid_values_count() { + // _total_value_count in header ignores of null values + return _total_values_remaining; + } + + Status set_data(Slice* slice) override { + _bit_reader.reset( + new BitReader((const uint8_t*)slice->data, cast_set(slice->size))); + RETURN_IF_ERROR(_init_header()); + _data = slice; + _offset = 0; + return Status::OK(); + } + + // Set BitReader which is already initialized by DeltaLengthByteArrayDecoder or + // DeltaByteArrayDecoder + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_init_header()); + return Status::OK(); + } + +private: + static constexpr int kMaxDeltaBitWidth = static_cast(sizeof(T) * 8); + Status _init_header(); + Status _init_block(); + Status _init_mini_block(int bit_width); + Status _get_internal(T* buffer, uint32_t max_values, uint32_t* out_num_values); + + std::vector _values; + + std::shared_ptr _bit_reader; + uint32_t _values_per_block; + uint32_t _mini_blocks_per_block; + uint32_t _values_per_mini_block; + uint32_t _total_value_count; + + T _min_delta; + T _last_value; + + uint32_t _mini_block_idx; + std::vector _delta_bit_widths; + int _delta_bit_width; + // If the page doesn't contain any block, `_block_initialized` will + // always be false. Otherwise, it will be true when first block initialized. + bool _block_initialized; + + uint32_t _total_values_remaining; + // Remaining values in current mini block. If the current block is the last mini block, + // _values_remaining_current_mini_block may greater than _total_values_remaining. + uint32_t _values_remaining_current_mini_block; +}; + +class DeltaLengthByteArrayDecoder final : public DeltaDecoder { +public: + explicit DeltaLengthByteArrayDecoder() + : _len_decoder(), _buffered_length(0), _buffered_data(0) {} + + Status skip_values(size_t num_values) override { + _values.resize(num_values); + int num_valid_values; + return _get_internal(_values.data(), cast_set(num_values), &num_valid_values); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + num_values, decoded_count); + } + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + + Status set_data(Slice* slice) override { + if (slice->size == 0) { + return Status::OK(); + } + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + _data = slice; + _offset = 0; + RETURN_IF_ERROR(_decode_lengths()); + return Status::OK(); + } + + Status set_bit_reader(std::shared_ptr bit_reader) { + _bit_reader = std::move(bit_reader); + RETURN_IF_ERROR(_decode_lengths()); + return Status::OK(); + } + +private: + // Decode all the encoded lengths. The decoder_ will be at the start of the encoded data + // after that. + Status _decode_lengths(); + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _len_decoder; + + int _num_valid_values; + uint32_t _length_idx; + std::vector _buffered_length; + std::vector _buffered_data; +}; + +class DeltaByteArrayDecoder : public DeltaDecoder { +public: + explicit DeltaByteArrayDecoder() : _buffered_prefix_length(0), _buffered_data(0) {} + + Status skip_values(size_t num_values) override { + _values.resize(num_values); + int num_valid_values; + return _get_internal(_values.data(), cast_set(num_values), &num_valid_values); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + _string_refs.resize(num_values); + for (size_t row = 0; row < num_values; ++row) { + _string_refs[row] = StringRef(_values[row].data, _values[row].size); + } + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(num_values)); + const size_t byte_size = num_values * static_cast(_type_length); + _fixed_values.resize(byte_size); + for (size_t row = 0; row < num_values; ++row) { + if (UNLIKELY(_values[row].size != static_cast(_type_length))) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + _values[row].size, _type_length); + } + memcpy(_fixed_values.data() + row * _type_length, _values[row].data, _type_length); + } + return consumer.consume(_fixed_values.data(), num_values, + static_cast(_type_length)); + } + + Status set_data(Slice* slice) override { + _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); + RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(_bit_reader)); + + // get the number of encoded prefix lengths + int num_prefix = _prefix_len_decoder.valid_values_count(); + // call _prefix_len_decoder.Decode to decode all the prefix lengths. + // all the prefix lengths are buffered in _buffered_prefix_length. + _buffered_prefix_length.resize(num_prefix); + uint32_t ret; + RETURN_IF_ERROR( + _prefix_len_decoder.decode(_buffered_prefix_length.data(), num_prefix, &ret)); + DCHECK_EQ(ret, num_prefix); + _prefix_len_offset = 0; + _num_valid_values = num_prefix; + + // at this time, the decoder_ will be at the start of the encoded suffix data. + RETURN_IF_ERROR(_suffix_decoder.set_bit_reader(_bit_reader)); + + // TODO: read corrupted files written with bug(PARQUET-246). _last_value should be set + // to _last_value_in_previous_page when decoding a new page(except the first page) + _last_value = ""; + return Status::OK(); + } + + Status decode(Slice* buffer, int num_values, int* out_num_values) { + return _get_internal(buffer, num_values, out_num_values); + } + +private: + Status _decode_slices(size_t num_values) { + _values.resize(num_values); + int decoded_count = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &decoded_count)); + if (UNLIKELY(decoded_count != num_values)) { + return Status::IOError("Expected {} Parquet delta-byte-array values, decoded {}", + num_values, decoded_count); + } + return Status::OK(); + } + + Status _get_internal(Slice* buffer, int max_values, int* out_num_values); + + std::vector _values; + std::vector _string_refs; + std::vector _fixed_values; + std::shared_ptr _bit_reader; + DeltaBitPackDecoder _prefix_len_decoder; + DeltaLengthByteArrayDecoder _suffix_decoder; + std::string _last_value; + // string buffer for last value in previous page + std::string _last_value_in_previous_page; + int _num_valid_values; + uint32_t _prefix_len_offset; + std::vector _buffered_prefix_length; + std::vector _buffered_data; +}; +} // namespace doris::format::parquet::native + +namespace doris::format::parquet::native { + +template +Status DeltaBitPackDecoder::_init_header() { + if (!_bit_reader->GetVlqInt(&_values_per_block) || + !_bit_reader->GetVlqInt(&_mini_blocks_per_block) || + !_bit_reader->GetVlqInt(&_total_value_count) || + !_bit_reader->GetZigZagVlqInt(&_last_value)) { + return Status::IOError("Init header eof"); + } + if (_values_per_block == 0) { + return Status::InvalidArgument("Cannot have zero value per block"); + } + if (_values_per_block % 128 != 0) { + return Status::InvalidArgument( + "the number of values in a block must be multiple of 128, but it's " + + std::to_string(_values_per_block)); + } + if (_mini_blocks_per_block == 0) { + return Status::InvalidArgument("Cannot have zero miniblock per block"); + } + _values_per_mini_block = _values_per_block / _mini_blocks_per_block; + if (_values_per_mini_block == 0) { + return Status::InvalidArgument("Cannot have zero value per miniblock"); + } + if (_values_per_mini_block % 32 != 0) { + return Status::InvalidArgument( + "The number of values in a miniblock must be multiple of 32, but it's " + + std::to_string(_values_per_mini_block)); + } + _total_values_remaining = _total_value_count; + _delta_bit_widths.resize(_mini_blocks_per_block); + // init as empty property + _block_initialized = false; + _values_remaining_current_mini_block = 0; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_block() { + DCHECK_GT(_total_values_remaining, 0) << "InitBlock called at EOF"; + if (!_bit_reader->GetZigZagVlqInt(&_min_delta)) { + return Status::IOError("Init block eof"); + } + + // read the bitwidth of each miniblock + uint8_t* bit_width_data = _delta_bit_widths.data(); + for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) { + if (!_bit_reader->GetAligned(1, bit_width_data + i)) { + return Status::IOError("Decode bit-width EOF"); + } + // Note that non-conformant bitwidth entries are allowed by the Parquet spec + // for extraneous miniblocks in the last block (GH-14923), so we check + // the bitwidths when actually using them (see InitMiniBlock()). + } + _mini_block_idx = 0; + _block_initialized = true; + RETURN_IF_ERROR(_init_mini_block(bit_width_data[0])); + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_init_mini_block(int bit_width) { + if (bit_width > kMaxDeltaBitWidth) [[unlikely]] { + return Status::InvalidArgument("delta bit width larger than integer bit width"); + } + _delta_bit_width = bit_width; + _values_remaining_current_mini_block = _values_per_mini_block; + return Status::OK(); +} + +template +Status DeltaBitPackDecoder::_get_internal(T* buffer, uint32_t num_values, + uint32_t* out_num_values) { + num_values = std::min(num_values, _total_values_remaining); + if (num_values == 0) { + *out_num_values = 0; + return Status::OK(); + } + uint32_t i = 0; + while (i < num_values) { + if (_values_remaining_current_mini_block == 0) [[unlikely]] { + if (!_block_initialized) [[unlikely]] { + buffer[i++] = _last_value; + DCHECK_EQ(i, 1); // we're at the beginning of the page + if (i == num_values) { + // When block is uninitialized and i reaches num_values we have two + // different possibilities: + // 1. _total_value_count == 1, which means that the page may have only + // one value (encoded in the header), and we should not initialize + // any block. + // 2. _total_value_count != 1, which means we should initialize the + // incoming block for subsequent reads. + if (_total_value_count != 1) { + RETURN_IF_ERROR(_init_block()); + } + break; + } + RETURN_IF_ERROR(_init_block()); + } else { + ++_mini_block_idx; + if (_mini_block_idx < _mini_blocks_per_block) { + RETURN_IF_ERROR(_init_mini_block(_delta_bit_widths.data()[_mini_block_idx])); + } else { + RETURN_IF_ERROR(_init_block()); + } + } + } + + uint32_t values_decode = std::min(_values_remaining_current_mini_block, num_values - i); + for (uint32_t j = 0; j < values_decode; ++j) { + if (!_bit_reader->GetValue(_delta_bit_width, buffer + i + j)) { + return Status::IOError("Get batch EOF"); + } + } + for (int j = 0; j < values_decode; ++j) { + // Addition between min_delta, packed int and last_value should be treated as + // unsigned addition. Overflow is as expected. + buffer[i + j] = static_cast(_min_delta) + static_cast(buffer[i + j]) + + static_cast(_last_value); + _last_value = buffer[i + j]; + } + _values_remaining_current_mini_block -= values_decode; + i += values_decode; + } + _total_values_remaining -= num_values; + + if (_total_values_remaining == 0) [[unlikely]] { + if (!_bit_reader->Advance(_delta_bit_width * _values_remaining_current_mini_block)) { + return Status::IOError("Skip padding EOF"); + } + _values_remaining_current_mini_block = 0; + } + *out_num_values = num_values; + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp new file mode 100644 index 00000000000000..1f1166421a4035 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_dict_decoder.hpp @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris::format::parquet::native { + +// Dictionary decoders retain only encoded physical values and the index-stream cursor. Logical +// interpretation is deliberately delegated to DataTypeSerDe through decode_dictionary(). +template +class FixLengthDictDecoder final : public BaseDictDecoder { +public: + FixLengthDictDecoder() = default; + ~FixLengthDictDecoder() override = default; + + size_t dictionary_size() const override { return _num_dictionary_values; } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return fixed_consumer.consume(_dict.get(), _num_dictionary_values, + static_cast(_type_length)); + } + + Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, + size_t num_values) override { + if (UNLIKELY(_type_length <= 0 || length < 0 || + num_values > std::numeric_limits::max() / + static_cast(_type_length) || + num_values * static_cast(_type_length) != + static_cast(length))) { + return Status::Corruption("Wrong dictionary data for fixed length type"); + } + if (UNLIKELY(dict == nullptr)) { + return Status::Corruption("Fixed-length Parquet dictionary is null"); + } + _dict = std::move(dict); + _num_dictionary_values = num_values; + ++_dictionary_generation; + return Status::OK(); + } + +private: + size_t _num_dictionary_values = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp new file mode 100644 index 00000000000000..463ef3db82a394 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" + +namespace doris::format::parquet::native { + +Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, + ParquetFixedValueConsumer& consumer) { + const size_t byte_size = num_values * static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet plain decoder"); + } + RETURN_IF_ERROR(consumer.consume(reinterpret_cast(_data->data) + _offset, + num_values, static_cast(_type_length))); + _offset += byte_size; + return Status::OK(); +} + +Status FixLengthPlainDecoder::skip_values(size_t num_values) { + _offset += _type_length * num_values; + if (UNLIKELY(_offset > _data->size)) { + return Status::IOError("Out-of-bounds access in parquet data decoder"); + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h new file mode 100644 index 00000000000000..e0f3962ed0bde5 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include "common/status.h" +#include "core/column/column_fixed_length_object.h" +#include "core/data_type/data_type.h" +#include "format/parquet/parquet_common.h" +#include "format_v2/parquet/reader/native/decoder.h" + +namespace doris { +class ColumnSelectVector; +} // namespace doris + +namespace doris::format::parquet::native { + +class FixLengthPlainDecoder final : public Decoder { +public: + FixLengthPlainDecoder() = default; + ~FixLengthPlainDecoder() override = default; + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + + Status skip_values(size_t num_values) override; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp new file mode 100644 index 00000000000000..40e435b9b55141 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_decoder.h" + +#include + +#include + +#include "common/cast_set.h" +#include "format/parquet/parquet_common.h" +#include "util/bit_stream_utils.inline.h" +#include "util/bit_util.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { + +static constexpr size_t V1_LEVEL_SIZE = 4; + +Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels) { + _encoding = encoding; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + switch (encoding) { + case tparquet::Encoding::RLE: { + if (slice->size < V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + + uint8_t* data = (uint8_t*)slice->data; + uint32_t num_bytes = decode_fixed32_le(data); + if (num_bytes > slice->size - V1_LEVEL_SIZE) { + return Status::Corruption("Wrong parquet level format"); + } + _rle_decoder = RleDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); + + slice->data += V1_LEVEL_SIZE + num_bytes; + slice->size -= V1_LEVEL_SIZE + num_bytes; + break; + } + case tparquet::Encoding::BIT_PACKED: { + uint32_t num_bits = num_levels * _bit_width; + uint32_t num_bytes = BitUtil::RoundUpNumBytes(num_bits); + if (num_bytes > slice->size) { + return Status::Corruption("Wrong parquet level format"); + } + _bit_packed_decoder = BitReader((uint8_t*)slice->data, num_bytes); + + slice->data += num_bytes; + slice->size -= num_bytes; + break; + } + default: + return Status::IOError("Unsupported encoding for parquet level"); + } + return Status::OK(); +} + +Status LevelDecoder::init_v2(const Slice& levels, level_t max_level, uint32_t num_levels) { + _encoding = tparquet::Encoding::RLE; + _bit_width = cast_set(BitUtil::log2(max_level + 1)); + _max_level = max_level; + _num_levels = num_levels; + size_t byte_length = levels.size; + _rle_decoder = + RleDecoder((uint8_t*)levels.data, cast_set(byte_length), _bit_width); + return Status::OK(); +} + +size_t LevelDecoder::get_levels(level_t* levels, size_t n) { + // toto template. + if (_encoding == tparquet::Encoding::RLE) { + n = std::min((size_t)_num_levels, n); + auto num_decoded = _rle_decoder.get_values(levels, n); + _num_levels -= num_decoded; + return num_decoded; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + n = std::min((size_t)_num_levels, n); + for (size_t i = 0; i < n; ++i) { + if (!_bit_packed_decoder.GetValue(_bit_width, &levels[i])) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "Failed to decode BIT_PACKED levels"); + } + } + _num_levels -= n; + return n; + } + return 0; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h new file mode 100644 index 00000000000000..7d91b10eb7cdde --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include + +#include "common/status.h" +#include "format/parquet/parquet_common.h" +#include "util/bit_stream_utils.h" +#include "util/rle_encoding.h" +#include "util/slice.h" + +namespace doris::format::parquet::native { +class LevelDecoder { +public: + LevelDecoder() = default; + ~LevelDecoder() = default; + + Status init(Slice* slice, tparquet::Encoding::type encoding, level_t max_level, + uint32_t num_levels); + + Status init_v2(const Slice& levels, level_t max_level, uint32_t num_levels); + + inline bool has_levels() const { return _num_levels > 0; } + + size_t get_levels(level_t* levels, size_t n); + + inline size_t get_next_run(level_t* val, size_t max_run) { + return _rle_decoder.GetNextRun(val, max_run); + } + + inline level_t get_next() { + level_t next = -1; + _rle_decoder.Get(&next); + return next; + } + + inline void rewind_one() { _rle_decoder.RewindOne(); } + + const RleDecoder& rle_decoder() const { return _rle_decoder; } + +private: + tparquet::Encoding::type _encoding; + level_t _bit_width = 0; + level_t _max_level = 0; + uint32_t _num_levels = 0; + RleDecoder _rle_decoder; + BitReader _bit_packed_decoder; +}; + +} // namespace doris::format::parquet::native \ No newline at end of file diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp new file mode 100644 index 00000000000000..16da9912cd4282 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/level_reader.h" + +#include +#include + +#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/buffered_reader.h" +#include "io/fs/tracing_file_reader.h" + +namespace doris::format::parquet::native { + +class LevelReader::Impl { +public: + virtual ~Impl() = default; + + virtual Status init() = 0; + virtual Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) = 0; + virtual ColumnChunkReaderStatistics statistics() = 0; +}; + +template +class LevelReaderImpl final : public LevelReader::Impl { +public: + LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, + size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, + bool enable_page_cache) + : _file(std::move(file)), + _column_chunk(std::move(column_chunk)), + _field(field), + _total_rows(total_rows), + _max_buffer_size(max_buffer_size), + _io_ctx(io_ctx), + _enable_page_cache(enable_page_cache) {} + + Status init() override { + DORIS_CHECK(_file != nullptr); + DORIS_CHECK(_field != nullptr); + const auto& metadata = _column_chunk.meta_data; + const int64_t chunk_start = has_dict_page(metadata) ? metadata.dictionary_page_offset + : metadata.data_page_offset; + const size_t chunk_size = metadata.total_compressed_size; + size_t prefetch_buffer_size = std::min(chunk_size, _max_buffer_size); + auto* tracing_reader = typeid_cast(_file.get()); + if ((tracing_reader != nullptr && + typeid_cast(tracing_reader->inner_reader().get()) != + nullptr) || + typeid_cast(_file.get()) != nullptr) { + prefetch_buffer_size = 0; + } + _stream = std::make_unique(_file, chunk_start, chunk_size, + prefetch_buffer_size); + _chunk_reader = std::make_unique>( + _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, + ParquetPageReadContext(_enable_page_cache)); + return _chunk_reader->init(); + } + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) override { + DORIS_CHECK(repetition_levels != nullptr); + DORIS_CHECK(definition_levels != nullptr); + DORIS_CHECK(rows_read != nullptr); + repetition_levels->clear(); + definition_levels->clear(); + *rows_read = 0; + if (_current_row > _total_rows || rows > _total_rows - _current_row) { + return Status::Corruption("Parquet level reader requested rows [{}, {}) of {}", + _current_row, _current_row + rows, _total_rows); + } + if constexpr (IN_COLLECTION) { + RETURN_IF_ERROR( + read_nested_rows(rows, repetition_levels, definition_levels, rows_read)); + } else { + RETURN_IF_ERROR(read_flat_rows(rows, repetition_levels, definition_levels, rows_read)); + } + _current_row += *rows_read; + return Status::OK(); + } + + ColumnChunkReaderStatistics statistics() override { return _chunk_reader->statistics(); } + +private: + Status read_flat_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + const size_t read_now = std::min( + rows - *rows_read, static_cast(_chunk_reader->remaining_num_values())); + if (read_now == 0) { + return Status::Corruption("Parquet flat level reader made no progress"); + } + RETURN_IF_ERROR( + _chunk_reader->read_levels(read_now, repetition_levels, definition_levels)); + *rows_read += read_now; + if (_chunk_reader->remaining_num_values() == 0 && _chunk_reader->has_next_page()) { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + return Status::OK(); + } + + Status read_nested_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + while (*rows_read < rows) { + RETURN_IF_ERROR(_chunk_reader->seek_to_nested_row(_current_row + *rows_read)); + const size_t start_level = definition_levels->size(); + size_t loaded_rows = 0; + bool crosses_page = false; + RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows( + *repetition_levels, rows - *rows_read, &loaded_rows, &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR(_chunk_reader->skip_nested_values(*definition_levels, start_level)); + while (crosses_page) { + const size_t continuation_start = definition_levels->size(); + RETURN_IF_ERROR(_chunk_reader->load_cross_page_nested_row(*repetition_levels, + &crosses_page)); + RETURN_IF_ERROR(_chunk_reader->fill_def(*definition_levels)); + RETURN_IF_ERROR( + _chunk_reader->skip_nested_values(*definition_levels, continuation_start)); + } + if (loaded_rows == 0) { + return Status::Corruption("Parquet nested level reader made no progress"); + } + *rows_read += loaded_rows; + } + return Status::OK(); + } + + io::FileReaderSPtr _file; + tparquet::ColumnChunk _column_chunk; + FieldSchema* _field = nullptr; + size_t _total_rows = 0; + size_t _max_buffer_size = 0; + io::IOContext* _io_ctx = nullptr; + bool _enable_page_cache = false; + size_t _current_row = 0; + std::unique_ptr _stream; + std::unique_ptr> _chunk_reader; +}; + +Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + FieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + std::unique_ptr* reader) { + DORIS_CHECK(reader != nullptr); + DORIS_CHECK(field != nullptr); + std::unique_ptr impl; + if (field->repetition_level > 0) { + impl = std::make_unique>(std::move(file), std::move(column_chunk), + field, total_rows, max_buffer_size, io_ctx, + enable_page_cache); + } else { + impl = std::make_unique>(std::move(file), std::move(column_chunk), + field, total_rows, max_buffer_size, io_ctx, + enable_page_cache); + } + RETURN_IF_ERROR(impl->init()); + reader->reset(new LevelReader(std::move(impl))); + return Status::OK(); +} + +LevelReader::LevelReader(std::unique_ptr impl) : _impl(std::move(impl)) {} + +LevelReader::~LevelReader() = default; + +Status LevelReader::read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read) { + return _impl->read_rows(rows, repetition_levels, definition_levels, rows_read); +} + +Status LevelReader::skip_rows(size_t rows) { + size_t rows_read = 0; + RETURN_IF_ERROR( + read_rows(rows, &_skip_repetition_levels, &_skip_definition_levels, &rows_read)); + if (rows_read != rows) { + return Status::Corruption("Parquet level reader skipped {} of {} rows", rows_read, rows); + } + return Status::OK(); +} + +ColumnChunkReaderStatistics LevelReader::statistics() { + return _impl->statistics(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h new file mode 100644 index 00000000000000..df5ae7931a021d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "common/status.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "io/fs/file_reader_writer_fwd.h" + +namespace doris { +struct FieldSchema; +namespace io { +struct IOContext; +} +} // namespace doris + +namespace doris::format::parquet::native { + +// A physical-leaf reader that advances all three Parquet streams (repetition levels, definition +// levels and encoded values) but retains only the two level streams. It is used for shape-only +// operations such as COUNT(nullable_col), where materializing a large BYTE_ARRAY value would be +// both unnecessary and potentially unbounded. +// +// This deliberately shares ColumnChunkReader with the value path. Page V1/V2 parsing, +// decompression, dictionary-page handling, page cache semantics and corruption checks therefore +// cannot drift between an aggregate shortcut and an ordinary scan. +class LevelReader { +public: + class Impl; + + static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + FieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, + std::unique_ptr* reader); + + ~LevelReader(); + + Status read_rows(size_t rows, std::vector* repetition_levels, + std::vector* definition_levels, size_t* rows_read); + Status skip_rows(size_t rows); + ColumnChunkReaderStatistics statistics(); + +private: + explicit LevelReader(std::unique_ptr impl); + + std::unique_ptr _impl; + // COUNT range gaps can call skip repeatedly as the adaptive row cap changes. Keep these + // throw-away level buffers on the persistent reader so only their logical sizes are reset. + std::vector _skip_repetition_levels; + std::vector _skip_definition_levels; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp new file mode 100644 index 00000000000000..14496d6e447665 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/page_reader.h" + +#include +#include +#include +#include + +#include + +#include "common/compiler_util.h" // IWYU pragma: keep +#include "common/config.h" +#include "format/parquet/parquet_common.h" +#include "io/fs/buffered_reader.h" +#include "runtime/runtime_profile.h" +#include "storage/cache/page_cache.h" +#include "util/slice.h" +#include "util/thrift_util.h" + +namespace doris { +namespace io { +struct IOContext; +} // namespace io +} // namespace doris + +namespace doris::format::parquet::native { +static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; + +void ParquetPageCacheKeyBuilder::init(const std::string& path, int64_t mtime) { + _file_key_prefix = fmt::format("{}::{}", path, mtime); +} + +template +PageReader::PageReader(io::BufferedStreamReader* reader, + io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, + const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index) + : _reader(reader), + _io_ctx(io_ctx), + _offset(offset), + _start_offset(offset), + _end_offset(offset + length), + _total_rows(total_rows), + _metadata(metadata), + _page_read_ctx(page_read_ctx), + _offset_index(offset_index) { + _next_header_offset = _offset; + _state = INITIALIZED; + _page_cache_key_builder.init(_reader->path(), _reader->mtime()); + + if constexpr (OFFSET_INDEX) { + _end_row = _offset_index->page_locations.size() >= 2 + ? _offset_index->page_locations[1].first_row_index + : _total_rows; + } +} + +template +Status PageReader::parse_page_header() { + if (_state == HEADER_PARSED) { + return Status::OK(); + } + if (UNLIKELY(_offset < _start_offset || _offset >= _end_offset)) { + return Status::IOError("Out-of-bounds Access"); + } + if (UNLIKELY(_offset != _next_header_offset)) { + return Status::IOError("Wrong header position, should seek to a page header first"); + } + if (UNLIKELY(_state != INITIALIZED)) { + return Status::IOError("Should skip or load current page to get next page"); + } + + _page_statistics.page_read_counter += 1; + + // Parse page header from file; header bytes are saved for possible cache insertion + const uint8_t* page_header_buf = nullptr; + size_t max_size = _end_offset - _offset; + size_t header_size = std::min(INIT_PAGE_HEADER_SIZE, max_size); + const size_t MAX_PAGE_HEADER_SIZE = config::parquet_header_max_size_mb << 20; + uint32_t real_header_size = 0; + + // Try a header-only lookup in the page cache. Cached pages store + // header + optional v2 levels + uncompressed payload, so we can + // parse the page header directly from the cached bytes and avoid + // a file read for the header. + if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && + StoragePageCache::instance() != nullptr) { + PageCacheHandle handle; + StoragePageCache::CacheKey key = make_page_cache_key(static_cast(_offset)); + if (StoragePageCache::instance()->lookup(key, &handle, segment_v2::DATA_PAGE)) { + // Parse header directly from cached data + _page_cache_handle = std::move(handle); + Slice s = _page_cache_handle.data(); + real_header_size = cast_set(s.size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + auto st = deserialize_thrift_msg(reinterpret_cast(s.data), + &real_header_size, true, &_cur_page_header); + if (!st.ok()) return st; + // Increment page cache counters for a true cache hit on header+payload + _page_statistics.page_cache_hit_counter += 1; + // Detect whether the cached payload is compressed or decompressed and record + bool is_cache_payload_decompressed = + should_cache_decompressed(&_cur_page_header, _metadata); + + if (is_cache_payload_decompressed) { + _page_statistics.page_cache_decompressed_hit_counter += 1; + } else { + _page_statistics.page_cache_compressed_hit_counter += 1; + } + + _is_cache_payload_decompressed = is_cache_payload_decompressed; + + if constexpr (OFFSET_INDEX == false) { + if (is_header_v2()) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } + + // Save header bytes for later use (e.g., to insert updated cache entries) + _header_buf.assign(s.data, s.data + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); + } else { + _page_statistics.page_cache_missing_counter += 1; + // Clear any existing cache handle on miss to avoid holding stale handle + _page_cache_handle = PageCacheHandle(); + } + } + // NOTE: page cache lookup for *decompressed* page data is handled in + // ColumnChunkReader::load_page_data(). PageReader should only be + // responsible for parsing the header bytes from the file and saving + // them in `_header_buf` for possible later insertion into the cache. + while (true) { + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + header_size = std::min(header_size, max_size); + { + SCOPED_RAW_TIMER(&_page_statistics.read_page_header_time); + RETURN_IF_ERROR(_reader->read_bytes(&page_header_buf, _offset, header_size, _io_ctx)); + } + real_header_size = cast_set(header_size); + SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + auto st = + deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header); + if (st.ok()) { + break; + } + if (_offset + header_size >= _end_offset || real_header_size > MAX_PAGE_HEADER_SIZE) { + return Status::IOError( + "Failed to deserialize parquet page header. offset: {}, " + "header size: {}, end offset: {}, real header size: {}", + _offset, header_size, _end_offset, real_header_size); + } + header_size <<= 2; + } + + if constexpr (OFFSET_INDEX == false) { + if (is_header_v2()) { + _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _end_row = _start_row + _cur_page_header.data_page_header.num_values; + } + } + + // Save header bytes for possible cache insertion later + _header_buf.assign(page_header_buf, page_header_buf + real_header_size); + _last_header_size = real_header_size; + _page_statistics.parse_page_header_num++; + _offset += real_header_size; + _next_header_offset = _offset + _cur_page_header.compressed_page_size; + _state = HEADER_PARSED; + return Status::OK(); +} + +template +Status PageReader::get_page_data(Slice& slice) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::IOError("Should generate page header first to load current page data"); + } + if (UNLIKELY(_io_ctx && _io_ctx->should_stop)) { + return Status::EndOfFile("stop"); + } + slice.size = _cur_page_header.compressed_page_size; + RETURN_IF_ERROR(_reader->read_bytes(slice, _offset, _io_ctx)); + _offset += slice.size; + _state = DATA_LOADED; + return Status::OK(); +} + +template class PageReader; +template class PageReader; +template class PageReader; +template class PageReader; + +} // namespace doris::format::parquet::native \ No newline at end of file diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h new file mode 100644 index 00000000000000..fe4dc52a2b69c3 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -0,0 +1,261 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include +#include + +#include "common/cast_set.h" +#include "common/config.h" +#include "common/status.h" +#include "format/parquet/parquet_common.h" +#include "storage/cache/page_cache.h" +#include "util/block_compression.h" +namespace doris { +class BlockCompressionCodec; + +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io + +} // namespace doris + +namespace doris { +namespace io { +class BufferedStreamReader; +struct IOContext; +} // namespace io +struct Slice; +} // namespace doris + +namespace doris::format::parquet::native { +/** + * Use to deserialize parquet page header, and get the page data in iterator interface. + */ + +// Session-level options for parquet page reading/caching. +struct ParquetPageReadContext { + bool enable_parquet_file_page_cache = true; + ParquetPageReadContext() = default; + ParquetPageReadContext(bool enable_parquet_file_page_cache) + : enable_parquet_file_page_cache(enable_parquet_file_page_cache) {} +}; + +inline bool should_cache_decompressed(const tparquet::PageHeader* header, + const tparquet::ColumnMetaData& metadata) { + if (header->compressed_page_size <= 0) return true; + if (metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) return true; + if (header->uncompressed_page_size == 0) return true; + + double ratio = static_cast(header->uncompressed_page_size) / + static_cast(header->compressed_page_size); + return ratio <= config::parquet_page_cache_decompress_threshold; +} + +class ParquetPageCacheKeyBuilder { +public: + void init(const std::string& path, int64_t mtime); + StoragePageCache::CacheKey make_key(uint64_t end_offset, int64_t offset) const { + return StoragePageCache::CacheKey(_file_key_prefix, end_offset, offset); + } + +private: + std::string _file_key_prefix; +}; + +template +class PageReader { +public: + struct PageStatistics { + int64_t decode_header_time = 0; + int64_t skip_page_header_num = 0; + int64_t parse_page_header_num = 0; + int64_t read_page_header_time = 0; + int64_t page_cache_hit_counter = 0; + int64_t page_cache_missing_counter = 0; + int64_t page_cache_compressed_hit_counter = 0; + int64_t page_cache_decompressed_hit_counter = 0; + int64_t page_cache_write_counter = 0; + int64_t page_cache_compressed_write_counter = 0; + int64_t page_cache_decompressed_write_counter = 0; + int64_t page_read_counter = 0; + }; + + PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, + uint64_t length, size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& page_read_ctx, + const tparquet::OffsetIndex* offset_index = nullptr); + ~PageReader() = default; + + bool has_next_page() const { + if constexpr (OFFSET_INDEX) { + return _page_index + 1 != _offset_index->page_locations.size(); + } else { + // Deprecated + // Parquet file may not be standardized, + // _end_offset may exceed the actual data area. + // ColumnChunkReader::has_next_page() use the number of parsed values for judgment + // ref:https://github.com/duckdb/duckdb/issues/10829 + // [[deprecated]] + LOG(FATAL) << "has_next_page should not be called when no offset index"; + return _offset < _end_offset; + } + } + + Status parse_page_header(); + + Status next_page() { + _page_statistics.skip_page_header_num += _state == INITIALIZED; + if constexpr (OFFSET_INDEX) { + _page_index++; + _start_row = _offset_index->page_locations[_page_index].first_row_index; + if (_page_index + 1 < _offset_index->page_locations.size()) { + _end_row = _offset_index->page_locations[_page_index + 1].first_row_index; + } else { + _end_row = _total_rows; + } + int64_t next_page_offset = _offset_index->page_locations[_page_index].offset; + _offset = next_page_offset; + _next_header_offset = next_page_offset; + _state = INITIALIZED; + } else { + if (UNLIKELY(_offset == _start_offset)) { + return Status::Corruption("should parse first page."); + } + + if (is_header_v2()) { + _start_row += _cur_page_header.data_page_header_v2.num_rows; + } else if constexpr (!IN_COLLECTION) { + _start_row += _cur_page_header.data_page_header.num_values; + } + + _offset = _next_header_offset; + _state = INITIALIZED; + } + + return Status::OK(); + } + + Status dict_next_page() { + if constexpr (OFFSET_INDEX) { + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + + Status get_page_header(const tparquet::PageHeader** page_header) { + if (UNLIKELY(_state != HEADER_PARSED)) { + return Status::InternalError("Page header not parsed"); + } + *page_header = &_cur_page_header; + return Status::OK(); + } + + Status get_page_data(Slice& slice); + + // Skip page data and update offset (used when data is loaded from cache) + void skip_page_data() { + if (_state == HEADER_PARSED) { + _offset += _cur_page_header.compressed_page_size; + _state = DATA_LOADED; + } + } + + const std::vector& header_bytes() const { return _header_buf; } + // header start offset for current page + int64_t header_start_offset() const { + return static_cast(_next_header_offset) - static_cast(_last_header_size) - + static_cast(_cur_page_header.compressed_page_size); + } + uint64_t file_end_offset() const { return _end_offset; } + bool cached_decompressed() const { + return should_cache_decompressed(&_cur_page_header, _metadata); + } + + PageStatistics& page_statistics() { return _page_statistics; } + + bool is_header_v2() { return _cur_page_header.__isset.data_page_header_v2; } + + // Returns whether the current page's cache payload is decompressed + bool is_cache_payload_decompressed() const { return _is_cache_payload_decompressed; } + + size_t start_row() const { return _start_row; } + + size_t end_row() const { return _end_row; } + + // Accessors for cache handle + bool has_page_cache_handle() const { return _page_cache_handle.cache() != nullptr; } + const doris::PageCacheHandle& page_cache_handle() const { return _page_cache_handle; } + StoragePageCache::CacheKey make_page_cache_key(int64_t offset) const { + return _page_cache_key_builder.make_key(_end_offset, offset); + } + +private: + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; + PageReaderState _state = INITIALIZED; + PageStatistics _page_statistics; + + io::BufferedStreamReader* _reader = nullptr; + io::IOContext* _io_ctx = nullptr; + // current reader offset in file location. + uint64_t _offset = 0; + // this page offset in file location. + uint64_t _start_offset = 0; + uint64_t _end_offset = 0; + uint64_t _next_header_offset = 0; + // current page row range + size_t _start_row = 0; + size_t _end_row = 0; + // total rows in this column chunk + size_t _total_rows = 0; + // Column metadata for this column chunk + const tparquet::ColumnMetaData& _metadata; + // Session-level parquet page cache options + ParquetPageReadContext _page_read_ctx; + // for page index + size_t _page_index = 0; + const tparquet::OffsetIndex* _offset_index; + + tparquet::PageHeader _cur_page_header; + bool _is_cache_payload_decompressed = true; + + // Page cache members + ParquetPageCacheKeyBuilder _page_cache_key_builder; + doris::PageCacheHandle _page_cache_handle; + // stored header bytes when cache miss so we can insert header+payload into cache + std::vector _header_buf; + // last parsed header size in bytes + uint32_t _last_header_size = 0; +}; + +template +std::unique_ptr> create_page_reader( + io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, uint64_t length, + size_t total_rows, const tparquet::ColumnMetaData& metadata, + const ParquetPageReadContext& ctx, const tparquet::OffsetIndex* offset_index = nullptr) { + return std::make_unique>( + reader, io_ctx, offset, length, total_rows, metadata, ctx, offset_index); +} + +} // namespace doris::format::parquet::native \ No newline at end of file diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 2f70d1acc8d96c..262d9f90da3661 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -119,8 +119,8 @@ void collect_projected_ids(const ParquetColumnSchema& schema, if (schema.kind == ParquetColumnSchemaKind::MAP) { DORIS_CHECK(!native_field.children.empty()); // MAP entry existence and offsets are owned by the key stream even for value-only - // projections. Keep the key reader live and validate key/value entry alignment in v1's - // native complex-column reader. + // projections. Keep the key reader live so the native complex reader can validate + // key/value entry alignment while constructing offsets. ids->insert(native_field.children[0].get_column_id()); } } @@ -241,10 +241,10 @@ Status NativeColumnReader::init( _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); native_runtime_state = _page_cache_runtime_state.get(); } - RETURN_IF_ERROR(::doris::ParquetColumnReader::create( - std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, - max_buffer_size, _offset_indexes, native_runtime_state, false, _projected_column_ids, - _filter_column_ids)); + RETURN_IF_ERROR(native::ColumnReader::create(std::move(file), field, row_group, _row_ranges, + timezone, io_ctx, _native_reader, max_buffer_size, + _offset_indexes, native_runtime_state, false, + _projected_column_ids, _filter_column_ids)); DORIS_CHECK(_native_reader != nullptr); _skip_column = _type->create_column(); return Status::OK(); @@ -521,7 +521,6 @@ int64_t NativeColumnReader::sync_native_profile() { } const auto stats = _native_reader->column_statistics(); const auto& reported = _reported_native_stats; - const auto& last_query = _last_native_query_stats; if (_profile.decompress_time != nullptr) { COUNTER_UPDATE(_profile.decompress_time, stats.decompress_time - reported.decompress_time); } @@ -552,7 +551,8 @@ int64_t NativeColumnReader::sync_native_profile() { COUNTER_UPDATE(_profile.convert_time, stats.convert_time - reported.convert_time); } if (_profile.materialization_time != nullptr) { - COUNTER_UPDATE(_profile.materialization_time, stats.convert_time - reported.convert_time); + COUNTER_UPDATE(_profile.materialization_time, + stats.materialization_time - reported.materialization_time); } if (_profile.page_index_read_calls != nullptr) { COUNTER_UPDATE(_profile.page_index_read_calls, @@ -570,12 +570,7 @@ int64_t NativeColumnReader::sync_native_profile() { COUNTER_UPDATE(_profile.read_page_header_time, stats.read_page_header_time - reported.read_page_header_time); } - // chunk_statistics() adds the PageReader's cumulative counters into its own accumulator on - // every query. The difference between two raw query results is therefore the PageReader's - // current cumulative value, not the increment since the previous query. -#define RECONSTRUCT_PAGE_STAT(field) (stats.field - last_query.field) - const int64_t page_read_count = RECONSTRUCT_PAGE_STAT(page_read_counter); - const int64_t page_read_delta = page_read_count - reported.page_read_counter; + const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; if (_profile.page_read_count != nullptr) { COUNTER_UPDATE(_profile.page_read_count, page_read_delta); } @@ -594,38 +589,24 @@ int64_t NativeColumnReader::sync_native_profile() { reported.page_cache_decompressed_write_counter); } if (_profile.page_cache_hit_count != nullptr) { - COUNTER_UPDATE( - _profile.page_cache_hit_count, - RECONSTRUCT_PAGE_STAT(page_cache_hit_counter) - reported.page_cache_hit_counter); + COUNTER_UPDATE(_profile.page_cache_hit_count, + stats.page_cache_hit_counter - reported.page_cache_hit_counter); } if (_profile.page_cache_miss_count != nullptr) { COUNTER_UPDATE(_profile.page_cache_miss_count, - RECONSTRUCT_PAGE_STAT(page_cache_missing_counter) - - reported.page_cache_missing_counter); + stats.page_cache_missing_counter - reported.page_cache_missing_counter); } if (_profile.page_cache_compressed_hit_count != nullptr) { COUNTER_UPDATE(_profile.page_cache_compressed_hit_count, - RECONSTRUCT_PAGE_STAT(page_cache_compressed_hit_counter) - + stats.page_cache_compressed_hit_counter - reported.page_cache_compressed_hit_counter); } if (_profile.page_cache_decompressed_hit_count != nullptr) { COUNTER_UPDATE(_profile.page_cache_decompressed_hit_count, - RECONSTRUCT_PAGE_STAT(page_cache_decompressed_hit_counter) - + stats.page_cache_decompressed_hit_counter - reported.page_cache_decompressed_hit_counter); } -#undef RECONSTRUCT_PAGE_STAT _reported_native_stats = stats; - _reported_native_stats.page_read_counter = page_read_count; - _reported_native_stats.page_cache_hit_counter = - stats.page_cache_hit_counter - last_query.page_cache_hit_counter; - _reported_native_stats.page_cache_missing_counter = - stats.page_cache_missing_counter - last_query.page_cache_missing_counter; - _reported_native_stats.page_cache_compressed_hit_counter = - stats.page_cache_compressed_hit_counter - last_query.page_cache_compressed_hit_counter; - _reported_native_stats.page_cache_decompressed_hit_counter = - stats.page_cache_decompressed_hit_counter - - last_query.page_cache_decompressed_hit_counter; - _last_native_query_stats = stats; return page_read_delta; } diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 783982d5c41d2a..c642d48d7dd0ea 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -27,10 +27,10 @@ #include #include "format/parquet/parquet_common.h" -#include "format/parquet/vparquet_column_reader.h" #include "format/table/table_schema_change_helper.h" #include "format_v2/column_data.h" #include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/parquet/reader/native/column_reader.h" namespace doris { class FileMetaData; @@ -104,7 +104,7 @@ class NativeColumnReader final : public ParquetColumnReader { std::set _filter_column_ids; std::unordered_map _offset_indexes; std::shared_ptr _schema_node; - std::unique_ptr<::doris::ParquetColumnReader> _native_reader; + std::unique_ptr _native_reader; std::unique_ptr _page_cache_runtime_state; std::vector _selected_ranges; size_t _selected_range_idx = 0; @@ -113,12 +113,9 @@ class NativeColumnReader final : public ParquetColumnReader { bool _dictionary_filter_enabled = false; bool _nested = false; - // Most native statistics are ordinary cumulative values. Page/cache statistics are special: - // v1 folds the PageReader's cumulative snapshot into ColumnChunkReader on every query. Keep the - // previous raw query so sync_native_profile() can reconstruct the real cumulative page value - // instead of reporting the same pages once per FileScannerV2 batch. - ::doris::ParquetColumnReader::ColumnStatistics _last_native_query_stats; - ::doris::ParquetColumnReader::ColumnStatistics _reported_native_stats; + // The native tree exposes cumulative statistics. Keep the last reported snapshot so each + // FileScannerV2 batch contributes only its delta to RuntimeProfile. + native::ColumnReader::ColumnStatistics _reported_native_stats; std::vector _filter_scratch; MutableColumnPtr _skip_column; MutableColumnPtr _dictionary_id_column; diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp new file mode 100644 index 00000000000000..f4379b2e60e692 --- /dev/null +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -0,0 +1,322 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include + +#include "core/assert_cast.h" +#include "core/column/column_decimal.h" +#include "core/column/column_string.h" +#include "core/column/column_vector.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/data_type/data_type_time.h" +#include "core/data_type/data_type_timestamptz.h" +#include "core/data_type/data_type_varbinary.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { +namespace { + +class TestParquetDecodeSource final : public ParquetDecodeSource { +public: + template + void set_fixed_values(const std::vector& values) { + _value_width = sizeof(T); + _fixed_values.resize(values.size() * sizeof(T)); + memcpy(_fixed_values.data(), values.data(), _fixed_values.size()); + } + + void set_fixed_bytes(std::vector values, size_t value_width) { + _fixed_values = std::move(values); + _value_width = value_width; + } + + void set_dictionary(std::vector values, size_t value_width, + std::vector indices) { + _dictionary = std::move(values); + _dictionary_width = value_width; + _indices = std::move(indices); + _index_offset = 0; + ++_dictionary_generation; + } + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_LE((_fixed_offset + num_values) * _value_width, _fixed_values.size()); + const uint8_t* begin = _fixed_values.data() + _fixed_offset * _value_width; + _fixed_offset += num_values; + return consumer.consume(begin, num_values, _value_width); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_LE(_binary_offset + num_values, _binary_refs.size()); + const StringRef* begin = _binary_refs.data() + _binary_offset; + _binary_offset += num_values; + return consumer.consume(begin, num_values); + } + + Status skip_values(size_t num_values) override { + _fixed_offset += num_values; + _binary_offset += num_values; + _index_offset += num_values; + return Status::OK(); + } + + bool has_dictionary() const override { return !_dictionary.empty(); } + uint64_t dictionary_generation() const override { return _dictionary_generation; } + size_t dictionary_size() const override { + return _dictionary_width == 0 ? 0 : _dictionary.size() / _dictionary_width; + } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + ++_dictionary_decode_calls; + return fixed_consumer.consume(_dictionary.data(), dictionary_size(), _dictionary_width); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + DORIS_CHECK_LE(_index_offset + num_values, _indices.size()); + indices->assign(_indices.begin() + _index_offset, + _indices.begin() + _index_offset + num_values); + _index_offset += num_values; + return Status::OK(); + } + + size_t dictionary_decode_calls() const { return _dictionary_decode_calls; } + +private: + std::vector _fixed_values; + std::vector _binary_refs; + std::vector _dictionary; + std::vector _indices; + size_t _value_width = 0; + size_t _dictionary_width = 0; + size_t _fixed_offset = 0; + size_t _binary_offset = 0; + size_t _index_offset = 0; + uint64_t _dictionary_generation = 0; + size_t _dictionary_decode_calls = 0; +}; + +TEST(DataTypeSerDeParquetTest, MaterializesLogicalUnsignedIntegersDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 0, 7}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::INTEGER, + .logical_integer_bit_width = 32, + .logical_integer_is_signed = false}; + ParquetMaterializationState state; + DataTypeInt64 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 4294967295LL); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 7); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFloat16Directly) { + TestParquetDecodeSource source; + source.set_fixed_values({0x3C00, 0xC000, 0x7C00}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::FLOAT16, + .type_length = 2, + .logical_float16 = true}; + ParquetMaterializationState state; + DataTypeFloat32 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_FLOAT_EQ(data[0], 1.0F); + EXPECT_FLOAT_EQ(data[1], -2.0F); + EXPECT_TRUE(std::isinf(data[2])); +} + +TEST(DataTypeSerDeParquetTest, RescalesFixedBinaryDecimalDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x04, 0xD2, 0xFB, 0x2E}, 2); // 12.34 and -12.34 at scale 2. + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 2, + .decimal_precision = 4, + .decimal_scale = 2}; + ParquetMaterializationState state; + DataTypeDecimal64 type(18, 3); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 12340); + EXPECT_EQ(data[1].value, -12340); +} + +TEST(DataTypeSerDeParquetTest, ReusesTypedDictionary) { + TestParquetDecodeSource source; + std::vector dictionary {'a', 'a', 'b', 'b'}; + source.set_dictionary(std::move(dictionary), 2, {1, 0, 1, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2}; + ParquetMaterializationState state; + DataTypeString type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + ASSERT_EQ(state.typed_dictionary->size(), 2); + EXPECT_EQ(column->get_data_at(0).to_string(), "bb"); + EXPECT_EQ(column->get_data_at(1).to_string(), "aa"); + EXPECT_EQ(column->get_data_at(2).to_string(), "bb"); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 1); + EXPECT_EQ(column->get_data_at(3).to_string(), "aa"); + + source.set_dictionary({'c', 'c'}, 2, {0}); + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(source.dictionary_decode_calls(), 2); + EXPECT_EQ(column->get_data_at(4).to_string(), "cc"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDictionaryIndicesWithoutValues) { + TestParquetDecodeSource source; + source.set_dictionary({'a', 'a', 'b', 'b'}, 2, {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .encoding = ParquetValueEncoding::DICTIONARY, + .type_length = 2, + .dictionary_index_only = true}; + ParquetMaterializationState state; + DataTypeString type; + auto column = ColumnInt32::create(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = column->get_data(); + ASSERT_EQ(data.size(), 3); + EXPECT_EQ(data[0], 1); + EXPECT_EQ(data[1], 0); + EXPECT_EQ(data[2], 1); + EXPECT_FALSE(state.typed_dictionary); +} + +TEST(DataTypeSerDeParquetTest, MaterializesDateDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, 1, -1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ParquetMaterializationState state; + DataTypeDateV2 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 2), "1969-12-31"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampUnitsAndNegativeEpochDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({0, -1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1969-12-31 23:59:59.999999"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000001"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimeUnitsDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({3723456789, -1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = ParquetTimeUnit::MICROS}; + ParquetMaterializationState state; + DataTypeTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(type.to_string(*column, 0), "01:02:03.456789"); + EXPECT_EQ(type.to_string(*column, 1), "-00:00:01.000001"); +} + +TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { + TestParquetDecodeSource source; + source.set_fixed_values({-1, 1000001}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1969); + EXPECT_EQ(data[0].microsecond(), 999999); + EXPECT_EQ(data[1].second(), 1); + EXPECT_EQ(data[1].microsecond(), 1); +} + +TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { + TestParquetDecodeSource source; + source.set_fixed_bytes({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, + 0x0C, 0x0D, 0x0E, 0x0F}, + 16); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::UUID, + .type_length = 16, + .logical_uuid = true}; + ParquetMaterializationState state; + DataTypeVarbinary type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->get_data_at(0).to_string(), std::string("\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F", + 16)); +} + +} // namespace +} // namespace doris diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp new file mode 100644 index 00000000000000..12c33fc5aecd85 --- /dev/null +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "core/custom_allocator.h" +#include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/decoder.h" +#include "util/coding.h" + +namespace doris::format::parquet::native { +namespace { + +class RejectFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return Status::InternalError("Unexpected fixed dictionary"); + } +}; + +class CaptureBinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef* values, size_t num_values) override { + refs.assign(values, values + num_values); + return Status::OK(); + } + + std::vector refs; +}; + +class CaptureFixedConsumer final : public ParquetFixedValueConsumer { +public: + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + if (width == 0) { + width = value_width; + } + DORIS_CHECK_EQ(width, value_width); + bytes.insert(bytes.end(), values, values + num_values * value_width); + return Status::OK(); + } + + template + std::vector values() const { + DORIS_CHECK_EQ(width, sizeof(T)); + DORIS_CHECK_EQ(bytes.size() % sizeof(T), 0); + std::vector result(bytes.size() / sizeof(T)); + memcpy(result.data(), bytes.data(), bytes.size()); + return result; + } + + size_t width = 0; + std::vector bytes; +}; + +std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type physical_type) { + auto node = ::parquet::schema::PrimitiveNode::Make("value", ::parquet::Repetition::REQUIRED, + physical_type); + return std::make_shared<::parquet::ColumnDescriptor>(node, 0, 0); +} + +DorisUniqueBufferPtr make_byte_array_dictionary(const std::vector& values, + int32_t* length) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + *length = static_cast(total_size); + auto dictionary = make_unique_buffer(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(dictionary.get() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(dictionary.get() + offset, value.data(), value.size()); + offset += value.size(); + } + return dictionary; +} + +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); + const uint8_t* dictionary_address = dictionary.get(); + ByteArrayDictDecoder decoder; + + ASSERT_TRUE(decoder.set_dict(dictionary, dictionary_length, 2).ok()); + EXPECT_EQ(dictionary.get(), nullptr); + + RejectFixedConsumer fixed_consumer; + CaptureBinaryConsumer binary_consumer; + ASSERT_TRUE(decoder.decode_dictionary(fixed_consumer, binary_consumer).ok()); + ASSERT_EQ(binary_consumer.refs.size(), 2); + EXPECT_EQ(binary_consumer.refs[0].to_string_view(), "alpha"); + EXPECT_EQ(binary_consumer.refs[1].to_string_view(), "beta"); + EXPECT_EQ(binary_consumer.refs[0].data, + reinterpret_cast(dictionary_address + sizeof(uint32_t))); + + // bit width 1, an RLE run of three values (header = 3 << 1), dictionary id 1. + char valid_indices[] = {1, 6, 1}; + Slice valid_slice(valid_indices, sizeof(valid_indices)); + ASSERT_TRUE(decoder.set_data(&valid_slice).ok()); + std::vector decoded_indices; + ASSERT_TRUE(decoder.decode_dictionary_indices(3, &decoded_indices).ok()); + EXPECT_EQ(decoded_indices, std::vector({1, 1, 1})); + + // bit width 2, one RLE value with dictionary id 3. Skipping still validates the encoded id so + // filter selection cannot hide a corrupt dictionary stream. + char invalid_indices[] = {2, 2, 3}; + Slice invalid_slice(invalid_indices, sizeof(invalid_indices)); + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetDecodeSource& source = decoder; + EXPECT_TRUE(source.skip_values(1).is()); + + Slice empty_indices; + EXPECT_TRUE(decoder.set_data(&empty_indices).is()); +} + +TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {11, 22, 33}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer integer_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, integer_consumer).ok()); + EXPECT_EQ(integer_consumer.values(), std::vector({22, 33})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0b10001101)}; + Slice plain_boolean_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_boolean_slice).ok()); + CaptureFixedConsumer plain_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(8, plain_boolean_consumer).ok()); + EXPECT_EQ(plain_boolean_consumer.values(), + std::vector({1, 0, 1, 1, 0, 0, 0, 1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_boolean[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_boolean, sizeof(rle_boolean)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + ASSERT_TRUE(decoder->skip_values(3).ok()); + CaptureFixedConsumer rle_boolean_consumer; + ASSERT_TRUE(decoder->decode_fixed_values(5, rle_boolean_consumer).ok()); + EXPECT_EQ(rle_boolean_consumer.values(), std::vector({1, 0, 0, 0, 1})); +} + +TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer first_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, first_integer).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer remaining_integers; + ASSERT_TRUE(decoder->decode_fixed_values(2, remaining_integers).ok()); + EXPECT_EQ(first_integer.values(), std::vector({100})); + EXPECT_EQ(remaining_integers.values(), std::vector({99, 1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(3, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 3); + EXPECT_EQ(consumer.refs[0].to_string_view(), "prefix-b"); + EXPECT_EQ(consumer.refs[1].to_string_view(), "other"); + EXPECT_EQ(consumer.refs[2].to_string_view(), "other-tail"); + } +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { + const std::vector values {1.0F, -2.5F, 3.25F}; + std::vector encoded(values.size() * sizeof(float)); + for (size_t row = 0; row < values.size(); ++row) { + const auto* bytes = reinterpret_cast(&values[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded[byte * values.size() + row] = bytes[byte]; + } + } + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(1).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(2, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); +} + +} // namespace +} // namespace doris::format::parquet::native diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index f5c0dc7c3b9ce0..72ab5e58034eea 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -49,7 +49,7 @@ class CursorColumnReader final : public ParquetColumnReader { CursorColumnReader() : ParquetColumnReader(int64_schema(), std::make_shared()) {} Status read(int64_t rows, MutableColumnPtr& column, int64_t* rows_read) override { - DORIS_CHECK(column != nullptr); + DORIS_CHECK(column); DORIS_CHECK(rows_read != nullptr); auto& values = assert_cast(*column); for (int64_t row = 0; row < rows; ++row) { diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 482913c6ff015b..d95f18fdcb9f55 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -1272,6 +1272,10 @@ TEST_F(NewParquetReaderTest, CountComplexColumnUsesShapeOnlyPath) { EXPECT_EQ(result.count, 4); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); EXPECT_EQ(profile.get_counter("MaterializationTime")->value(), 0); + ASSERT_NE(profile.get_counter("PageReadCount"), nullptr); + EXPECT_GT(profile.get_counter("PageReadCount")->value(), 0); + ASSERT_NE(profile.get_counter("ParsePageHeaderNum"), nullptr); + EXPECT_GT(profile.get_counter("ParsePageHeaderNum")->value(), 0); } TEST_F(NewParquetReaderTest, CountArrayColumnUsesLevelsOnlyPath) { @@ -1426,8 +1430,8 @@ TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchC EXPECT_EQ(id_values.get_element(0), 1); EXPECT_EQ(id_values.get_element(4), 4); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); - EXPECT_EQ(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); ASSERT_NE(profile.get_counter("NestedBatches"), nullptr); @@ -1479,8 +1483,8 @@ TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { EXPECT_EQ(fixed_values.get_data_at(1).to_string(), std::string("\0x\0y", 4)); EXPECT_EQ(fixed_values.get_data_at(4).to_string(), std::string("\xff\x00\x7f\x80", 4)); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); - EXPECT_EQ(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); ASSERT_NE(profile.get_counter("ConvertTime"), nullptr); ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); @@ -1866,7 +1870,7 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { ASSERT_NE(profile.get_counter("ReaderReadRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSelectRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowReadRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); ASSERT_NE(profile.get_counter("MaterializationTime"), nullptr); ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); ASSERT_NE(profile.get_counter("FileFooterReadCalls"), nullptr); @@ -1883,7 +1887,7 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { EXPECT_EQ(profile.get_counter("ReaderReadRows")->value(), ROW_COUNT + 3); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 2); EXPECT_EQ(profile.get_counter("ReaderSelectRows")->value(), 3); - EXPECT_EQ(profile.get_counter("ArrowReadRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); EXPECT_GT(profile.get_counter("MaterializationTime")->value(), 0); EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); EXPECT_EQ(profile.get_counter("FileFooterReadCalls")->value() + @@ -2829,7 +2833,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { ASSERT_NE(profile.get_counter("SelectedRows"), nullptr); ASSERT_NE(profile.get_counter("RangeGapSkippedRows"), nullptr); ASSERT_NE(profile.get_counter("ReaderSkipRows"), nullptr); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); ASSERT_NE(profile.get_counter("RowGroupFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexFilterTime"), nullptr); ASSERT_NE(profile.get_counter("PageIndexReadTime"), nullptr); @@ -2839,7 +2843,7 @@ TEST_F(NewParquetReaderTest, PageIndexFilteredGapFlushesPendingOutputSkipOnce) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 64); EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); EXPECT_EQ(profile.get_counter("ReaderSkipRows")->value(), 0); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); EXPECT_GT(profile.get_counter("RowGroupFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexFilterTime")->value(), 0); EXPECT_GT(profile.get_counter("PageIndexReadTime")->value(), 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 36cc99ebf106c0..5532fe57dd2b5e 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -1238,8 +1238,8 @@ TEST_F(ParquetScanTest, FullyFilteredRowGroupsDropPendingLazyReaders) { EXPECT_EQ(total_rows, 0); EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 6); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); - ASSERT_NE(profile.get_counter("ArrowSkipRecordsTime"), nullptr); - EXPECT_EQ(profile.get_counter("ArrowSkipRecordsTime")->value(), 0); + ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); + EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); } // Scenario: row group 0 is fully filtered and leaves two pending lazy rows. Reset must discard that diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index b05ea798ee46b0..9b38ebb9dc70ff 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -107,6 +107,36 @@ format-specific checklist when reviewing Parquet or ORC. selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the optimization overhead and the avoided work; a low pruning ratio alone is not a defect. +## Parquet Native Decode Boundary + +- V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the + v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. +- Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. + Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, + builders, `DecodedColumnView`, or another decoded leaf batch. +- Verify physical/logical metadata is immutable per leaf reader and complete for signed integers, + decimal precision/scale, date/time/timestamp units and UTC adjustment, INT96, UUID, FLOAT16, and + fixed-width binary. Unsupported combinations return explicit errors before plausible output. +- Verify schema-change routing separately from physical decode. Integer, FLOAT-to-DOUBLE, decimal, + and string-family changes should use the direct target-SerDe path. Other supported logical casts + may use one persistent generic `ColumnTypeConverter` source column; its value/null-map sizes must + reset per batch while capacity is retained, and it must never become a decoder-facing ABI. +- Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload + ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate + every index before access, and invalidate cached dictionary state at Row Group/file/type changes. +- Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, + DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded + values without allocating output; null runs must append defaults without advancing payload. +- Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, + null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct + without materializing an intermediate complex column. +- For a STRUCT whose projected children are all missing after schema evolution, require a + levels-only physical reference leaf. It must advance and validate encoded payload cursors while + deriving the synthetic child count, without constructing a discarded string/complex column. +- `CountColumnReader` must use the native levels-only reader and must not decode payload or call + Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, + value decode, SerDe materialization, filtered-value skips, and page fragmentation. + ## Parquet Multi-Level Filtering - Use [FileScannerV2 Parquet Scan Design](file-scanner-v2-parquet-scan-design.md) as the detailed diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 92fd83eb26f90e..abf297e734726c 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -68,29 +68,28 @@ writes directly into Doris columns. This follows the same broad separation used planning is distinct from a persistent per-column reader, and page/encoding decoders expose narrow cursor-based contracts rather than an Arrow array as an intermediate result. -The migration is deliberately incremental because the existing v1 native kernel already contains -mature Parquet page and encoding support. The first production step reuses that kernel behind a -v2-owned adapter; decoder logic is copied into the v2 tree only when v2 must diverge from it: +V1 remains the differential baseline, but v2 owns an independent reader implementation. Its page +and encoding algorithms started from proven Doris behavior and are maintained under the v2 tree; +the production v2 path never instantiates the v1 `ParquetColumnReader`: | Stage | State and boundary | | --- | --- | -| Native encoding kernel | The current adapter calls the unchanged Doris native kernel used by v1, preserving its complete type/encoding matrix and page-cache behavior. Any decoder change is reimplemented under `be/src/format_v2/parquet/`; v1 remains unchanged. | +| Native encoding kernel | V2 owns the Column Chunk, page, level, and encoding decoders under `be/src/format_v2/parquet/reader/native/`. Decoders expose raw fixed/binary spans, validated dictionary indices, and skip operations; they never write Doris columns. | +| Logical materialization | `DataTypeSerDe` interprets Parquet physical/logical metadata and writes decoded spans directly into the final Doris column. Typed dictionaries and scratch are persistent per leaf reader. | | Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | -| Complex reconstruction | Build one shared Dremel level plan per requested parent-row range and use it for STRUCT/ARRAY/MAP siblings, offsets, and null maps. | +| Complex reconstruction | Choose a Dremel shape-owning leaf per requested parent-row range; derive parent offsets/nulls once from it and keep sibling leaf streams aligned to the same parent rows. | | Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | | Compatibility removal | Remove Arrow data-read adapters after type/encoding/page/writer compatibility and performance gates pass. Production v2 never falls back from a selected native reader to Arrow; an unsupported combination returns an explicit error. | -At the current migration boundary, ordinary value scans no longer use Arrow `RecordReader`, arrays, -or builders. Arrow remains in footer/schema planning, dictionary probing, and the existing -levels-only aggregate path. It is not a runtime fallback: after an ordinary column selects the -native reader, decode errors are returned directly. The production target eventually removes the -remaining Arrow metadata/aggregate objects; tests may also use Arrow as a fixture writer or oracle. +Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or +builders. Arrow remains in footer/schema planning and dictionary/statistics probing. It is not a +runtime fallback: after a column selects the native reader, decode errors are returned directly. +The production target eventually removes the remaining Arrow metadata objects; tests may also use +Arrow as a fixture writer or oracle. All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness -and performance control. Reusing its stable native kernel gives v2 the same physical/logical type, -encoding, malformed-input, conversion, and page-cache behavior without duplicating thousands of -lines before an actual semantic change is required. When such a change is required, the affected -decoder is reimplemented in the v2 tree instead of modifying v1. +and performance control. Compatibility is demonstrated through differential tests and the explicit +type/encoding matrix, not through a runtime dependency on v1 code. ### 2.2 Native Interface Ownership @@ -116,9 +115,9 @@ flowchart LR - **Decoder contract:** Consume a known number of logical level entries and encoded payload values, then materialize only selected rows. Decoders do not decide table projection, predicate meaning, or parent complex offsets. -- **Complex plan contract:** Definition/repetition levels are parsed once into parent-row boundaries - and child-presence/null decisions. All children consume the same plan so their offsets and null - maps cannot drift. +- **Complex plan contract:** The shape-owning leaf's definition/repetition levels are parsed once + into parent-row boundaries and child-presence/null decisions. Sibling physical streams consume + the same parent-row range and validate their counts so offsets and null maps cannot drift. ## 3. From File Open to Scan Plan @@ -370,10 +369,10 @@ selection, an empty selection, and an arbitrary fragmented selection use the sam Selection inputs are borrowed only for the duration of the decode call. For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes -definition-level runs, builds the four-way selection plan without an O(batch rows) action array, -and dispatches the plan to the active encoding decoder. A filtered non-null value must still advance -the encoding state even when it is not copied. This is the central invariant that prevents the next -batch from decoding shifted values. +definition-level runs, builds the four-way selection plan in a persistent action buffer whose +capacity survives adaptive batch changes, and dispatches its runs to the active encoding decoder. A +filtered non-null value must still advance the encoding state even when it is not copied. This is +the central invariant that prevents the next batch from decoding shifted values. ### 7.2 Page and Encoding Kernel @@ -399,17 +398,25 @@ payloads, integer overflow, and invalid lengths/IDs are part of the unit-test ma ### 7.3 Direct Materialization and Scratch Reuse -Decoders write directly into Doris mutable columns whenever physical and target layouts are -compatible. Fixed-width primitives reserve destination capacity once and append contiguous selected -runs. String-like decoders gather selected `StringRef` values into persistent scratch and perform -one batched append, so fragmented predicates do not cause one destination growth or copy call per -run. Scratch stores references only while the backing page/dictionary buffer is stable. +Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected +`DataTypeSerDe` consumes those spans and writes directly into the Doris mutable column. Fixed-width +types append contiguous runs; string-like paths gather `StringRef` values in persistent decoder +scratch before a batched append. References remain valid only while the page or dictionary buffer +is pinned by the persistent leaf reader. + +The direct path covers identical logical types, string-family compatibility, decimal +precision/scale changes, integer changes, and FLOAT-to-DOUBLE widening. Less common table-schema +changes such as STRING-to-DATE or DECIMAL-to-STRING keep the generic `ColumnTypeConverter`: the +file logical SerDe first fills a reusable source Doris column, then the generic cast appends to the +requested column. +This compatibility path is deliberately separate from the decoder ABI and does not reintroduce a +physical-value batch or `PhysicalToLogicalConverter` into ordinary Parquet reads. `DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded physical values and is useful to generic format conversion code, but routing every Parquet value -through it would recreate an intermediate materialization layer. A native encoding decoder consumes -encoded page bytes plus levels/selection and appends to the Doris physical column. Only a genuine -physical-to-logical mismatch invokes the reusable conversion layer after decode. +through it would recreate an intermediate materialization layer. The v2 native path does not use it: +ColumnReader handles levels/selection, Decoder parses encoding streams, and DataTypeSerDe performs +physical/logical conversion while appending to the final column. Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian two's-complement values with correct sign extension, and apply precision/scale conversion exactly @@ -422,11 +429,11 @@ definition/repetition levels, binary references, dictionary state, decompression column capacity. Logical sizes are reset at batch boundaries while capacity is retained. The native path does not create an Arrow builder or Arrow array. -### 7.4 Complex Types and Shared Level Plans +### 7.4 Complex Types and Parent Shape Plans Repeated Parquet leaves cannot interpret a requested parent-row count as a leaf-value count. One parent row may contain zero, one, or many level entries, and its final entry can reside on the next -page. The complex reader therefore builds a shared level plan with: +page. The complex reader therefore builds a parent shape plan from one owning leaf with: - parent-row start/end boundaries derived from repetition levels; - ancestor-null, collection-null, empty-collection, element-null, and present-value decisions from @@ -434,10 +441,12 @@ page. The complex reader therefore builds a shared level plan with: - child payload positions and selected parent rows; - cross-page continuation state for an unfinished parent row. -ARRAY and MAP offsets and null maps are derived from that plan. STRUCT children reuse the plan from -a representative present leaf and advance in parent-row lockstep; a missing or fully projected-out -child is materialized from the same parent count. MAP key/value readers must produce identical -entry counts, and Parquet's non-null key requirement is validated rather than repaired. +ARRAY and MAP offsets and null maps are derived from that plan. Parquet stores separate level +streams for separate physical leaves, so sibling readers still consume their own streams; they must +advance over the same parent-row range and validate their payload counts instead of redefining the +parent shape. STRUCT uses a representative present leaf for its null map and parent count. MAP uses +the key leaf as the entry-shape owner, requires the value leaf to produce the same entry count, and +validates Parquet's non-null key requirement rather than repairing it. Level scratch is sized by decoded level entries, not by the 16-bit parent batch cap. Long repeated rows and long null/non-null runs are split into representable internal runs without introducing a @@ -458,19 +467,22 @@ conversion, and complex-column state and materializes the complete result direct Doris column. No Arrow value reader, intermediate decoded leaf container, temporary nested Doris column, or public load-before-build protocol participates in predicate or output scans. -Internally, complex decoding still has to solve the shared-level-plan problem. For example, levels +Internally, complex decoding still has to solve the parent-shape problem. For example, levels representing `[["a", "b"], NULL, []]` produce entry counts `[2, 0, 0]`, parent nulls `[0, 1, 0]`, and string payload ordinals `[0, 1]`. MAP uses the key leaf as the entry-shape owner and validates the value leaf against it; STRUCT children advance in parent-row lockstep. This state belongs behind the native reader boundary rather than in caller-visible phases. -`CountColumnReader` is the only data-page compatibility exception. Existing -`COUNT(nullable_col)` pushdown needs definition/repetition levels but Arrow does not expose its -level decoder independently of `RecordReader`. The adapter therefore selects one representative -leaf (the key for MAP), calls `ReadRecords`, copies only levels, and immediately releases any binary -builder chunks. It exposes neither decoded values nor the ordinary scan-reader API, so it cannot -become an Arrow fallback. This preserves the existing pushdown semantics while keeping large complex -values out of the retained aggregate state. +When schema evolution makes every projected STRUCT child missing, the native reader retains one +physical reference leaf solely for shape. Its levels-only interface advances definition, +repetition, and encoded payload cursors (including dictionary-index validation), counts STRUCT +instances from the parent thresholds, and discards the payload without allocating a temporary +string or nested Doris column. + +`CountColumnReader` selects one representative leaf (the key for MAP) and uses the v2 native +`LevelReader` to consume definition/repetition levels without decoding payload values. It exposes +neither decoded values nor the ordinary scan-reader API, so COUNT pushdown cannot become a value +fallback and large complex payloads never enter aggregate state. Doris v1 remains the behavior/performance baseline: `read_column_data()` owns physical decode and the collection reader consumes persistent level buffers. DuckDB provides the same useful design @@ -607,9 +619,10 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun - When the base reader is CachedRemoteFileReader, predicate/output ranges for the current Row Group may be prefetched into FileCache. -- The remaining Arrow metadata/index path may use MergeRangeFileReader. The current native - BufferedFileStreamReader is not routed through that Arrow wrapper; it uses v1's stream/page cache - path while background prefetch warms the same Doris FileCache blocks. +- After dictionary probing, small projected chunks share one Row-Group-scoped + MergeRangeFileReader on the native data-page path. Large chunks and in-memory files keep the base + reader. The remaining Arrow metadata/index adapter has an independent wrapper and never owns the + native decoder's stream cursor. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. From ae4a199ae4389d7f51c782bde81a1893ce58a52f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 20:05:33 +0800 Subject: [PATCH 06/34] [refactor](be) Simplify Parquet v2 decoder factory ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: Split the Parquet v2 decoder factory by encoding family and replace raw allocations with std::make_unique. This keeps decoder construction behavior unchanged while satisfying function-size and ownership static-analysis rules. ### Release note None ### Check List (For Author) - Test: Unit Test - Remote ASAN BE build and 90 Parquet v2/SerDe unit tests - Remote clang-tidy on the changed decoder factory - Behavior changed: No - Does this need documentation: No --- .../parquet/reader/native/decoder.cpp | 200 +++++++++--------- 1 file changed, 102 insertions(+), 98 deletions(-) diff --git a/be/src/format_v2/parquet/reader/native/decoder.cpp b/be/src/format_v2/parquet/reader/native/decoder.cpp index c42f09af75d18e..5f6a978317bde7 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/decoder.cpp @@ -30,123 +30,127 @@ #include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" namespace doris::format::parquet::native { +namespace { +Status unsupported_type(tparquet::Type::type type, tparquet::Encoding::type encoding) { + return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", + tparquet::to_string(type), tparquet::to_string(encoding)); +} + +Status create_plain_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::PLAIN); + } +} + +Status create_dictionary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::BOOLEAN: + return Status::InternalError("Boolean type cannot have a dictionary page"); + case tparquet::Type::BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT96: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FLOAT: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::DOUBLE: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique>(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::RLE_DICTIONARY); + } +} + +Status create_delta_binary_decoder(tparquet::Type::type type, std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + decoder = std::make_unique>(); + return Status::OK(); + case tparquet::Type::INT64: + decoder = std::make_unique>(); + return Status::OK(); + default: + return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); + } +} + +Status create_byte_stream_split_decoder(tparquet::Type::type type, + std::unique_ptr& decoder) { + switch (type) { + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::INT96: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + decoder = std::make_unique(); + return Status::OK(); + default: + return unsupported_type(type, tparquet::Encoding::BYTE_STREAM_SPLIT); + } +} +} // namespace + Status Decoder::get_decoder(tparquet::Type::type type, tparquet::Encoding::type encoding, std::unique_ptr& decoder) { switch (encoding) { case tparquet::Encoding::PLAIN: - switch (type) { - case tparquet::Type::BOOLEAN: - decoder.reset(new BoolPlainDecoder()); - break; - case tparquet::Type::BYTE_ARRAY: - decoder.reset(new ByteArrayPlainDecoder()); - break; - case tparquet::Type::INT32: - case tparquet::Type::INT64: - case tparquet::Type::INT96: - case tparquet::Type::FLOAT: - case tparquet::Type::DOUBLE: - case tparquet::Type::FIXED_LEN_BYTE_ARRAY: - decoder.reset(new FixLengthPlainDecoder()); - break; - default: - return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", - tparquet::to_string(type), tparquet::to_string(encoding)); - } - break; + return create_plain_decoder(type, decoder); case tparquet::Encoding::RLE_DICTIONARY: - switch (type) { - case tparquet::Type::BOOLEAN: - return Status::InternalError("Bool type can't has dictionary page"); - case tparquet::Type::BYTE_ARRAY: - decoder.reset(new ByteArrayDictDecoder()); - break; - case tparquet::Type::INT32: - decoder.reset(new FixLengthDictDecoder()); - break; - case tparquet::Type::INT64: - decoder.reset(new FixLengthDictDecoder()); - break; - case tparquet::Type::INT96: - decoder.reset(new FixLengthDictDecoder()); - break; - case tparquet::Type::FLOAT: - decoder.reset(new FixLengthDictDecoder()); - break; - case tparquet::Type::DOUBLE: - decoder.reset(new FixLengthDictDecoder()); - break; - case tparquet::Type::FIXED_LEN_BYTE_ARRAY: - decoder.reset(new FixLengthDictDecoder()); - break; - default: - return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", - tparquet::to_string(type), tparquet::to_string(encoding)); - } - break; + return create_dictionary_decoder(type, decoder); case tparquet::Encoding::RLE: - switch (type) { - case tparquet::Type::BOOLEAN: - decoder.reset(new BoolRLEDecoder()); - break; - default: - return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", - tparquet::to_string(type), tparquet::to_string(encoding)); + if (type != tparquet::Type::BOOLEAN) { + return unsupported_type(type, encoding); } - break; + decoder = std::make_unique(); + return Status::OK(); case tparquet::Encoding::DELTA_BINARY_PACKED: - // Supports only INT32 and INT64. - switch (type) { - case tparquet::Type::INT32: - decoder.reset(new DeltaBitPackDecoder()); - break; - case tparquet::Type::INT64: - decoder.reset(new DeltaBitPackDecoder()); - break; - default: - return Status::InternalError("DELTA_BINARY_PACKED only supports INT32 and INT64"); - } - break; + return create_delta_binary_decoder(type, decoder); case tparquet::Encoding::DELTA_BYTE_ARRAY: - switch (type) { - case tparquet::Type::BYTE_ARRAY: - case tparquet::Type::FIXED_LEN_BYTE_ARRAY: - decoder.reset(new DeltaByteArrayDecoder()); - break; - default: + if (type != tparquet::Type::BYTE_ARRAY && type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) { return Status::InternalError( "DELTA_BYTE_ARRAY only supports BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY."); } - break; + decoder = std::make_unique(); + return Status::OK(); case tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY: - switch (type) { - case tparquet::Type::BYTE_ARRAY: - decoder.reset(new DeltaLengthByteArrayDecoder()); - break; - default: + if (type != tparquet::Type::BYTE_ARRAY) { return Status::InternalError("DELTA_LENGTH_BYTE_ARRAY only supports BYTE_ARRAY."); } - break; + decoder = std::make_unique(); + return Status::OK(); case tparquet::Encoding::BYTE_STREAM_SPLIT: - switch (type) { - case tparquet::Type::INT32: - case tparquet::Type::INT64: - case tparquet::Type::INT96: - case tparquet::Type::FLOAT: - case tparquet::Type::DOUBLE: - case tparquet::Type::FIXED_LEN_BYTE_ARRAY: - decoder.reset(new ByteStreamSplitDecoder()); - break; - default: - return Status::InternalError("Unsupported type {}(encoding={}) in parquet decoder", - tparquet::to_string(type), tparquet::to_string(encoding)); - } - break; + return create_byte_stream_split_decoder(type, decoder); default: return Status::InternalError("Unsupported encoding {}(type={}) in parquet decoder", tparquet::to_string(encoding), tparquet::to_string(type)); } - return Status::OK(); } -} // namespace doris::format::parquet::native \ No newline at end of file +} // namespace doris::format::parquet::native From 2df2cfeaa0942cc7aa34fe34e79300ca7409aa4a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 20:24:20 +0800 Subject: [PATCH 07/34] [fix](be) Bound Parquet v2 nested scratch retention ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: Persistent Parquet v2 leaf readers reused level, null, selection, conversion, and dictionary-index scratch across batches. A legal repeated column can expand one logical batch into an exceptional number of leaf values, so staggered outliers could retain large allocations in multiple leaves until Row Group teardown. Keep normal-size capacity reusable, recursively release individual scratch buffers above a 4 MiB high-water mark after the top-level complex reader consumes the level plan, and cover the retained-capacity drop with a unit test. ### Release note None ### Check List (For Author) - Test: Unit Test - Remote ASAN ./build.sh --be - Remote Parquet v2 and SerDe unit-test suite - Behavior changed: No - Does this need documentation: Yes. Updated the FileScannerV2 Parquet design and review guide. --- .../parquet/reader/native/column_reader.cpp | 63 +++++++++++++++++++ .../parquet/reader/native/column_reader.h | 34 +++++++++- .../parquet/reader/native_column_reader.cpp | 3 + .../format_v2/parquet/native_decoder_test.cpp | 17 +++++ docs/file-scanner-v2-code-review-guide.md | 6 +- docs/file-scanner-v2-parquet-scan-design.md | 7 ++- 6 files changed, 125 insertions(+), 5 deletions(-) diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 927ba73866c4c8..c28a0e48cd2d75 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -83,6 +83,20 @@ bool is_direct_decimal_type(PrimitiveType type) { } } +template +bool release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + DORIS_CHECK(values != nullptr); + if (values->capacity() * sizeof(T) <= max_retained_bytes) { + return false; + } + std::vector().swap(*values); + return true; +} + +size_t retained_set_bytes(const std::unordered_set& values) { + return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); +} + // The target SerDe can fuse physical decode with these logical type changes. Less common schema // changes retain the generic file-format converter as a compatibility path: the decoder still // exposes raw spans, but the source SerDe first materializes a reusable source column before the @@ -474,6 +488,54 @@ Status ScalarColumnReader::init(io::FileReaderSPtr return Status::OK(); } +template +void ScalarColumnReader::release_batch_scratch( + size_t max_retained_bytes) { + bool release_selection = false; + release_selection |= release_vector_if_oversized(&_rep_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_def_levels, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_null_run_lengths, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_nested_filter_map_data, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.dictionary_indices, + max_retained_bytes); + if (retained_set_bytes(_ancestor_null_indices) > max_retained_bytes) { + std::unordered_set().swap(_ancestor_null_indices); + release_selection = true; + } + if (release_selection) { + _select_vector = ColumnSelectVector(); + } + if (_logical_conversion_scratch_bytes > max_retained_bytes) { + _logical_converter.reset(); + _converter_source_type = nullptr; + _converter_target_type = nullptr; + _logical_conversion_scratch_bytes = 0; + } +} + +#ifdef BE_TEST +template +void ScalarColumnReader::reserve_batch_scratch_for_test( + size_t elements) { + _rep_levels.reserve(elements); + _def_levels.reserve(elements); + _null_run_lengths.reserve(elements); + _nested_filter_map_data.reserve(elements); + _materialization_state.dictionary_indices.reserve(elements); + _ancestor_null_indices.reserve(elements); +} + +template +size_t ScalarColumnReader::retained_batch_scratch_bytes_for_test() + const { + return _rep_levels.capacity() * sizeof(level_t) + _def_levels.capacity() * sizeof(level_t) + + _null_run_lengths.capacity() * sizeof(uint16_t) + + _nested_filter_map_data.capacity() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + retained_set_bytes(_ancestor_null_indices); +} +#endif + template Status ScalarColumnReader::_skip_values(size_t num_values) { if (num_values == 0) { @@ -889,6 +951,7 @@ Status ScalarColumnReader::read_column_data( } SCOPED_RAW_TIMER(&_convert_time); RETURN_IF_ERROR(_logical_converter->convert(converted_source_column, converted_column)); + _logical_conversion_scratch_bytes = converted_source_column->allocated_bytes(); doris_column = std::move(converted_column); return Status::OK(); }; diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 847d9d35204077..98b7e2c4e6c62e 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -184,6 +184,11 @@ class ColumnReader { virtual ColumnStatistics column_statistics() = 0; virtual void close() = 0; + // A repeated parent can expand one logical-row batch into millions of leaf values. Keep + // ordinary batch scratch for reuse, but let the top-level adapter release exceptional + // high-water allocations after every parent offset/null-map consumer has finished. + virtual void release_batch_scratch(size_t max_retained_bytes) = 0; + virtual void reset_filter_map_index() = 0; FieldSchema* get_field_schema() const { return _field_schema; } @@ -238,6 +243,13 @@ class ScalarColumnReader : public ColumnReader { } void close() override {} + void release_batch_scratch(size_t max_retained_bytes) override; + +#ifdef BE_TEST + void reserve_batch_scratch_for_test(size_t elements); + size_t retained_batch_scratch_bytes_for_test() const; +#endif + void reset_filter_map_index() override { _filter_map_index = 0; // nested } @@ -321,14 +333,15 @@ class ScalarColumnReader : public ColumnReader { ParquetDecodeContext _decode_context; ParquetMaterializationState _materialization_state; bool _dictionary_index_only = false; - // Batch scratch is retained by the persistent leaf reader. Only logical sizes are reset, so - // adaptive batch-size changes and repeated complex reads do not allocate fresh level plans. + // Normal-size batch scratch is retained by the persistent leaf reader. Oversized allocations + // are released only after the top-level parent has consumed this leaf's level plan. std::vector _null_run_lengths; std::unordered_set _ancestor_null_indices; std::vector _nested_filter_map_data; FilterMap _nested_filter_map; ColumnSelectVector _select_vector; int64_t _convert_time = 0; + size_t _logical_conversion_scratch_bytes = 0; Status _skip_values(size_t num_values); Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, @@ -364,6 +377,10 @@ class ArrayColumnReader : public ColumnReader { ColumnStatistics column_statistics() override { return _element_reader->column_statistics(); } void close() override {} + void release_batch_scratch(size_t max_retained_bytes) override { + _element_reader->release_batch_scratch(max_retained_bytes); + } + void reset_filter_map_index() override { _element_reader->reset_filter_map_index(); } private: @@ -403,6 +420,11 @@ class MapColumnReader : public ColumnReader { void close() override {} + void release_batch_scratch(size_t max_retained_bytes) override { + _key_reader->release_batch_scratch(max_retained_bytes); + _value_reader->release_batch_scratch(max_retained_bytes); + } + void reset_filter_map_index() override { _key_reader->reset_filter_map_index(); _value_reader->reset_filter_map_index(); @@ -466,6 +488,12 @@ class StructColumnReader : public ColumnReader { void close() override {} + void release_batch_scratch(size_t max_retained_bytes) override { + for (const auto& reader : _child_readers) { + reader.second->release_batch_scratch(max_retained_bytes); + } + } + void reset_filter_map_index() override { for (const auto& reader : _child_readers) { reader.second->reset_filter_map_index(); @@ -564,6 +592,8 @@ class SkipReadingReader : public ColumnReader { // Nothing to close for skip reading } + void release_batch_scratch(size_t) override {} + void reset_filter_map_index() override { _filter_map_index = 0; } }; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 262d9f90da3661..849963a6666908 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -43,6 +43,8 @@ namespace doris::format::parquet { namespace { +constexpr size_t MAX_RETAINED_BATCH_SCRATCH_BYTES = 4UL << 20; + DataTypePtr projected_type(const ParquetColumnSchema& schema, const format::LocalColumnIndex* projection) { if (!format::is_partial_projection(projection)) { @@ -298,6 +300,7 @@ Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_ if (_nested && _profile.nested_batches != nullptr) { COUNTER_UPDATE(_profile.nested_batches, 1); } + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); if (*rows_read != rows) { return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", *rows_read, rows, _name); diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 12c33fc5aecd85..9c981c36c8467d 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -27,6 +27,7 @@ #include "core/custom_allocator.h" #include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" +#include "format_v2/parquet/reader/native/column_reader.h" #include "format_v2/parquet/reader/native/decoder.h" #include "util/coding.h" @@ -249,5 +250,21 @@ TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); } +TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchIsReleased) { + ::doris::RowRanges row_ranges; + tparquet::ColumnChunk chunk; + ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); + + constexpr size_t max_retained_bytes = 64UL << 10; + reader.reserve_batch_scratch_for_test(1UL << 16); + const size_t oversized_bytes = reader.retained_batch_scratch_bytes_for_test(); + ASSERT_GT(oversized_bytes, max_retained_bytes); + + reader.release_batch_scratch(max_retained_bytes); + const size_t released_bytes = reader.retained_batch_scratch_bytes_for_test(); + EXPECT_LT(released_bytes, oversized_bytes); + EXPECT_LE(released_bytes, max_retained_bytes + sizeof(void*)); +} + } // namespace } // namespace doris::format::parquet::native diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index 9b38ebb9dc70ff..91d28a769deca4 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -120,7 +120,8 @@ format-specific checklist when reviewing Parquet or ORC. - Verify schema-change routing separately from physical decode. Integer, FLOAT-to-DOUBLE, decimal, and string-family changes should use the direct target-SerDe path. Other supported logical casts may use one persistent generic `ColumnTypeConverter` source column; its value/null-map sizes must - reset per batch while capacity is retained, and it must never become a decoder-facing ABI. + reset per batch while normal-size capacity is retained, oversized capacity must be released after + the top-level parent consumes the batch, and it must never become a decoder-facing ABI. - Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate every index before access, and invalidate cached dictionary state at Row Group/file/type changes. @@ -130,6 +131,9 @@ format-specific checklist when reviewing Parquet or ORC. - Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct without materializing an intermediate complex column. +- Require a bounded high-water policy for persistent definition/repetition, null, selection, + conversion, and dictionary-index scratch. Test that an oversized repeated-value batch releases + retained capacity without discarding ordinary reusable capacity. - For a STRUCT whose projected children are all missing after schema evolution, require a levels-only physical reference leaf. It must advance and validate encoded payload cursors while deriving the synthetic child count, without constructing a discarded string/complex column. diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index abf297e734726c..62bafc4fbd17ef 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -426,8 +426,11 @@ checks prove the result is equivalent. The persistent leaf reader owns reusable conversion objects, null map, selection ranges, definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris -column capacity. Logical sizes are reset at batch boundaries while capacity is retained. The native -path does not create an Arrow builder or Arrow array. +column capacity. Logical sizes are reset at batch boundaries and normal-size capacity is retained. +After the top-level complex reader has consumed the level plan, any individual batch scratch buffer +above the 4 MiB high-water limit is released; this prevents staggered repeated-value outliers from +accumulating one retained allocation per leaf for the rest of the Row Group. The native path does +not create an Arrow builder or Arrow array. ### 7.4 Complex Types and Parent Shape Plans From c9a5d6cbcf1bc17068f77942df2fd11e3a87bb5f Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 16 Jul 2026 22:54:13 +0800 Subject: [PATCH 08/34] [improvement](be) Batch sparse Parquet v2 decoding ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: Highly selective Parquet v2 scans entered DataTypeSerDe and the encoding decoder once per selection run. Fragmented selections therefore amplified virtual dispatch, consumer construction, and cursor-management overhead. Add a page-fragment hybrid selection plan that enters SerDe once and lets each native encoding batch gather or compact selected values while preserving and validating the full physical cursor. Add profile counters, decoder coverage, scratch retention bounds, and design/review documentation. ### Release note None ### Check List (For Author) - Test: No test run (per user request) - Behavior changed: Yes (sparse Parquet v2 materialization is batched per page fragment) - Does this need documentation: Yes (included in this PR) --- .../data_type_serde/parquet_decode_source.h | 65 +++++++ be/src/format_v2/parquet/parquet_profile.cpp | 9 + be/src/format_v2/parquet/parquet_profile.h | 6 + .../reader/native/bool_plain_decoder.cpp | 22 +++ .../reader/native/bool_plain_decoder.h | 10 +- .../reader/native/bool_rle_decoder.cpp | 17 ++ .../parquet/reader/native/bool_rle_decoder.h | 8 +- .../native/byte_array_plain_decoder.cpp | 27 +++ .../reader/native/byte_array_plain_decoder.h | 3 + .../native/byte_stream_split_decoder.cpp | 30 ++++ .../reader/native/byte_stream_split_decoder.h | 3 + .../reader/native/column_chunk_reader.cpp | 83 +++++++++ .../reader/native/column_chunk_reader.h | 3 + .../parquet/reader/native/column_reader.cpp | 4 + .../parquet/reader/native/column_reader.h | 13 ++ .../format_v2/parquet/reader/native/decoder.h | 29 +++ .../reader/native/delta_bit_pack_decoder.h | 89 +++++++++- .../native/fix_length_plain_decoder.cpp | 32 ++++ .../reader/native/fix_length_plain_decoder.h | 8 + .../parquet/reader/native_column_reader.cpp | 13 ++ .../format_v2/parquet/native_decoder_test.cpp | 168 ++++++++++++++++++ docs/file-scanner-v2-code-review-guide.md | 11 +- docs/file-scanner-v2-parquet-scan-design.md | 37 ++-- 23 files changed, 671 insertions(+), 19 deletions(-) diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index bd7d93d96ee8e1..6a0e448d1f78d0 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -22,6 +22,7 @@ #include #include +#include "common/check.h" #include "common/status.h" #include "core/column/column.h" #include "core/string_ref.h" @@ -110,6 +111,21 @@ class ParquetBinaryValueConsumer { virtual Status consume(const StringRef* values, size_t num_values) = 0; }; +// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs are +// intentionally excluded: the native ColumnReader uses this plan only when the batch has no NULL +// leaf slots, so selected values can be appended in one pass without a temporary nullable column. +// Ranges are sorted, disjoint, and expressed in the physical value stream's coordinate space. +struct ParquetSelectionRange { + size_t first = 0; + size_t count = 0; +}; + +struct ParquetSelection { + size_t total_values = 0; + size_t selected_values = 0; + std::vector ranges; +}; + // Encoding decoders implement this interface. They own encoded-stream cursors and dictionary // storage, but they never know the destination Doris column type. DataTypeSerDe owns the consumer // and therefore the physical/logical-to-Doris conversion. @@ -122,6 +138,35 @@ class ParquetDecodeSource { ParquetBinaryValueConsumer& consumer) = 0; virtual Status skip_values(size_t num_values) = 0; + // Batch-level sparse decode. The default implementation preserves every encoding's cursor + // semantics while moving SerDe dispatch and consumer construction out of the selection-run + // loop. Decoders with cheap random access or batch decode override these methods to remove the + // remaining per-range virtual calls as well. + virtual Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_fixed_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + virtual Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_binary_values(range.count, consumer)); + cursor = range.first + range.count; + } + return skip_values(selection.total_values - cursor); + } + virtual bool has_dictionary() const { return false; } virtual uint64_t dictionary_generation() const { return 0; } virtual size_t dictionary_size() const { return 0; } @@ -132,6 +177,25 @@ class ParquetDecodeSource { virtual Status decode_dictionary_indices(size_t num_values, std::vector* indices) { return Status::NotSupported("Parquet dictionary indices are not supported by this decoder"); } + virtual Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) { + DORIS_CHECK(indices != nullptr); + indices->clear(); + indices->reserve(selection.selected_values); + std::vector range_indices; + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + DORIS_CHECK(range.first + range.count <= selection.total_values); + RETURN_IF_ERROR(skip_values(range.first - cursor)); + RETURN_IF_ERROR(decode_dictionary_indices(range.count, &range_indices)); + indices->insert(indices->end(), range_indices.begin(), range_indices.end()); + cursor = range.first + range.count; + } + RETURN_IF_ERROR(skip_values(selection.total_values - cursor)); + DORIS_CHECK_EQ(indices->size(), selection.selected_values); + return Status::OK(); + } }; // Dictionary values are materialized once into the selected Doris type. The state belongs to a @@ -139,6 +203,7 @@ class ParquetDecodeSource { struct ParquetMaterializationState { MutableColumnPtr typed_dictionary; std::vector dictionary_indices; + ParquetSelection selection; uint64_t dictionary_generation = std::numeric_limits::max(); void reset_dictionary() { diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 676c6631f084fc..597f86010a2798 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -79,6 +79,12 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_TIMER_WITH_LEVEL(profile, "LevelOnlySkipTime", parquet_profile, 1); materialization_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "MaterializationTime", parquet_profile, 1); + hybrid_selection_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionBatches", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "HybridSelectionRanges", + TUnit::UNIT, parquet_profile, 1); + hybrid_selection_null_fallback_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "HybridSelectionNullFallbackBatches", TUnit::UNIT, parquet_profile, 1); native_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativeReadCalls", TUnit::UNIT, parquet_profile, 1); native_page_fragments = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NativePageFragments", @@ -214,6 +220,9 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .level_only_read_time = level_only_read_time, .level_only_skip_time = level_only_skip_time, .materialization_time = materialization_time, + .hybrid_selection_batches = hybrid_selection_batches, + .hybrid_selection_ranges = hybrid_selection_ranges, + .hybrid_selection_null_fallback_batches = hybrid_selection_null_fallback_batches, .decompress_time = decompress_time, .decompress_count = decompress_cnt, .decode_header_time = decode_header_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index ef2cb452667278..a1a79973f446e3 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -38,6 +38,9 @@ struct ParquetColumnReaderProfile { RuntimeProfile::Counter* level_only_read_time = nullptr; RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; // value materialization time (ns) + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; // Native page/encoding reader internals. These counters intentionally mirror v1 so a v1/v2 // profile comparison attributes page IO, decompression, levels, value decode and conversion to // the same stages. @@ -140,6 +143,9 @@ struct ParquetProfile { RuntimeProfile::Counter* level_only_read_time = nullptr; RuntimeProfile::Counter* level_only_skip_time = nullptr; RuntimeProfile::Counter* materialization_time = nullptr; + RuntimeProfile::Counter* hybrid_selection_batches = nullptr; + RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; + RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; RuntimeProfile::Counter* native_read_calls = nullptr; RuntimeProfile::Counter* native_page_fragments = nullptr; RuntimeProfile::Counter* page_crossing_batches = nullptr; diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp index a585fe8d4234c2..d5c9d2f2ff6d4a 100644 --- a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -47,6 +47,28 @@ Status BoolPlainDecoder::decode_fixed_values(size_t num_values, return Status::OK(); } +Status BoolPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + selected_values_.resize(selection.selected_values); + size_t range_index = 0; + size_t output = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + bool value = false; + if (UNLIKELY(!_decode_value(&value))) { + return Status::IOError("Can't read enough booleans in plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + selected_values_[output++] = static_cast(value); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(selected_values_.data(), output, sizeof(uint8_t)); +} + Status BoolPlainDecoder::skip_values(size_t num_values) { int skip_cached = std::min(num_unpacked_values_ - unpacked_value_idx_, cast_set(num_values)); diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h index c44c4934d8ce30..7a6806d4acd91f 100644 --- a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -17,8 +17,9 @@ #pragma once -#include -#include +#include +#include +#include #include "common/compiler_util.h" // IWYU pragma: keep #include "common/status.h" @@ -53,6 +54,9 @@ class BoolPlainDecoder final : public Decoder { Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + Status skip_values(size_t num_values) override; protected: @@ -86,6 +90,8 @@ class BoolPlainDecoder final : public Decoder { /// Bit packed decoder, used if 'encoding_' is PLAIN. BatchedBitReader bool_values_; + + std::vector selected_values_; }; } // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp index 12322e05b033ac..d1923c83c9f8bc 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -64,4 +65,20 @@ Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueC return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); } +Status BoolRLEDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + _values.resize(selection.total_values); + if (!_decoder.get_values(_values.data(), selection.total_values)) { + return Status::IOError("Can't read enough booleans in Parquet RLE selection decoder"); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, + range.count * sizeof(uint8_t)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_values.data(), output, sizeof(uint8_t)); +} + } // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h index 502e724b359425..e05a33d5e77e8b 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -17,9 +17,8 @@ #pragma once -#include -#include - +#include +#include #include #include "common/status.h" @@ -42,6 +41,9 @@ class BoolRLEDecoder final : public Decoder { Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + Status skip_values(size_t num_values) override; private: diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp index df756abe2c904a..904d77ba33de9b 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -52,6 +52,33 @@ Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, return consumer.consume(_binary_values.data(), _binary_values.size()); } +Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) { + _binary_values.clear(); + _binary_values.reserve(selection.selected_values); + size_t range_index = 0; + for (size_t row = 0; row < selection.total_values; ++row) { + uint32_t length = 0; + RETURN_IF_ERROR(read_length(_data, &_offset, &length)); + if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { + return Status::IOError("Can't read enough bytes in Parquet plain selection decoder"); + } + while (range_index < selection.ranges.size() && + row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + ++range_index; + } + if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { + _binary_values.emplace_back(_data->data + _offset, length); + } + _offset += length; + } + DORIS_CHECK_EQ(_binary_values.size(), selection.selected_values); + if (_binary_values.empty()) { + return Status::OK(); + } + return consumer.consume(_binary_values.data(), _binary_values.size()); +} + Status ByteArrayPlainDecoder::skip_values(size_t num_values) { for (int i = 0; i < num_values; ++i) { uint32_t length = 0; diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h index 962981c8c77c8c..77ba88875ed5c5 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -47,6 +47,9 @@ class ByteArrayPlainDecoder final : public Decoder { Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override; + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override; + Status skip_values(size_t num_values) override; private: diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp index 6fda6d0d752a0a..783834cbfa7c2e 100644 --- a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/reader/native/byte_stream_split_decoder.h" #include +#include #include "core/column/column_fixed_length_object.h" #include "util/byte_stream_split.h" @@ -38,6 +39,35 @@ Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, return consumer.consume(_decoded_values.data(), num_values, static_cast(_type_length)); } +Status ByteStreamSplitDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet byte-stream-split selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError( + "Out-of-bounds access in Parquet byte-stream-split selection decoder"); + } + DORIS_CHECK_EQ(_data->size % value_width, 0); + const int64_t stride = static_cast(_data->size / value_width); + _decoded_values.resize(selection.selected_values * value_width); + size_t output = 0; + const size_t first_row = _offset / value_width; + for (const auto& range : selection.ranges) { + byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, + first_row + range.first, range.count, stride, + _decoded_values.data() + output * value_width); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + _offset += input_bytes; + return consumer.consume(_decoded_values.data(), output, value_width); +} + Status ByteStreamSplitDecoder::skip_values(size_t num_values) { _offset += _type_length * num_values; if (UNLIKELY(_offset > _data->size)) { diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h index ae68f9c470b2bb..d6d190742939d6 100644 --- a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -29,6 +29,9 @@ class ByteStreamSplitDecoder final : public Decoder { Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + Status skip_values(size_t num_values) override; private: diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index c250c4bc18a955..fae204b4c6e277 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -113,6 +113,80 @@ Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decod return Status::OK(); } +// Presents one sparse page request as an ordinary sequential source to DataTypeSerDe. SerDe is +// entered once per page fragment; the concrete decoder decides whether to gather selected spans, +// batch-decode and compact, or use the cursor-preserving range fallback. +class SelectedDecodeSource final : public ParquetDecodeSource { +public: + SelectedDecodeSource(Decoder& decoder, const ParquetSelection& selection) + : _decoder(decoder), _selection(selection) {} + + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_fixed_values(_selection, consumer); + } + + Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_binary_values(_selection, consumer); + } + + Status skip_values(size_t num_values) override { + return Status::NotSupported("Selected Parquet source cannot be skipped, values={}", + num_values); + } + + bool has_dictionary() const override { return _decoder.has_dictionary(); } + uint64_t dictionary_generation() const override { return _decoder.dictionary_generation(); } + size_t dictionary_size() const override { return _decoder.dictionary_size(); } + + Status decode_dictionary(ParquetFixedValueConsumer& fixed_consumer, + ParquetBinaryValueConsumer& binary_consumer) override { + return _decoder.decode_dictionary(fixed_consumer, binary_consumer); + } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_indices(_selection, indices); + } + +private: + Decoder& _decoder; + const ParquetSelection& _selection; +}; + +Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = select_vector.num_values(); + selection.selected_values = 0; + + size_t cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + DORIS_CHECK(read_type == ColumnSelectVector::CONTENT || + read_type == ColumnSelectVector::FILTERED_CONTENT); + if (read_type == ColumnSelectVector::CONTENT) { + selection.ranges.push_back({.first = cursor, .count = run_length}); + selection.selected_values += run_length; + } + cursor += run_length; + } + DORIS_CHECK_EQ(cursor, selection.total_values); + if (selection.selected_values == 0) { + return decoder.skip_values(selection.total_values); + } + + SCOPED_RAW_TIMER(materialization_time); + SelectedDecodeSource selected_source(decoder, selection); + return serde.read_column_from_parquet(column, selected_source, context, + selection.selected_values, state); +} + } // namespace template @@ -662,6 +736,15 @@ Status ColumnChunkReader::materialize_values( _remaining_num_values -= select_vector.num_values(); RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); if (select_vector.has_filter()) { + if (select_vector.num_nulls() == 0) { + ++_chunk_statistics.hybrid_selection_batches; + const Status status = decode_selected_non_null_values( + *doris_column, serde, *_page_decoder, context, state, select_vector, + &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + return status; + } + ++_chunk_statistics.hybrid_selection_null_fallback_batches; return decode_selected_values(*doris_column, serde, *_page_decoder, context, state, select_vector, &_chunk_statistics.materialization_time); } diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 653a7cc19cedd8..3cc5dc35910380 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -57,6 +57,9 @@ struct ColumnChunkReaderStatistics { int64_t decode_header_time = 0; int64_t decode_value_time = 0; int64_t materialization_time = 0; + int64_t hybrid_selection_batches = 0; + int64_t hybrid_selection_ranges = 0; + int64_t hybrid_selection_null_fallback_batches = 0; int64_t decode_dict_time = 0; int64_t decode_level_time = 0; int64_t skip_page_header_num = 0; diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index c28a0e48cd2d75..77ca37bbaecdb7 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -498,6 +498,8 @@ void ScalarColumnReader::release_batch_scratch( release_selection |= release_vector_if_oversized(&_nested_filter_map_data, max_retained_bytes); release_selection |= release_vector_if_oversized(&_materialization_state.dictionary_indices, max_retained_bytes); + release_selection |= release_vector_if_oversized(&_materialization_state.selection.ranges, + max_retained_bytes); if (retained_set_bytes(_ancestor_null_indices) > max_retained_bytes) { std::unordered_set().swap(_ancestor_null_indices); release_selection = true; @@ -522,6 +524,7 @@ void ScalarColumnReader::reserve_batch_scratch_for_ _null_run_lengths.reserve(elements); _nested_filter_map_data.reserve(elements); _materialization_state.dictionary_indices.reserve(elements); + _materialization_state.selection.ranges.reserve(elements); _ancestor_null_indices.reserve(elements); } @@ -532,6 +535,7 @@ size_t ScalarColumnReader::retained_batch_scratch_b _null_run_lengths.capacity() * sizeof(uint16_t) + _nested_filter_map_data.capacity() * sizeof(uint8_t) + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + retained_set_bytes(_ancestor_null_indices); } #endif diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 98b7e2c4e6c62e..e7ca4093707c2f 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -59,6 +59,9 @@ class ColumnReader { decode_header_time(0), decode_value_time(0), materialization_time(0), + hybrid_selection_batches(0), + hybrid_selection_ranges(0), + hybrid_selection_null_fallback_batches(0), decode_dict_time(0), decode_level_time(0), decode_null_map_time(0), @@ -83,6 +86,9 @@ class ColumnReader { decode_header_time(cs.decode_header_time), decode_value_time(cs.decode_value_time), materialization_time(cs.materialization_time), + hybrid_selection_batches(cs.hybrid_selection_batches), + hybrid_selection_ranges(cs.hybrid_selection_ranges), + hybrid_selection_null_fallback_batches(cs.hybrid_selection_null_fallback_batches), decode_dict_time(cs.decode_dict_time), decode_level_time(cs.decode_level_time), decode_null_map_time(null_map_time), @@ -105,6 +111,9 @@ class ColumnReader { int64_t decode_header_time; int64_t decode_value_time; int64_t materialization_time; + int64_t hybrid_selection_batches; + int64_t hybrid_selection_ranges; + int64_t hybrid_selection_null_fallback_batches; int64_t decode_dict_time; int64_t decode_level_time; int64_t decode_null_map_time; @@ -128,6 +137,10 @@ class ColumnReader { decode_header_time += col_statistics.decode_header_time; decode_value_time += col_statistics.decode_value_time; materialization_time += col_statistics.materialization_time; + hybrid_selection_batches += col_statistics.hybrid_selection_batches; + hybrid_selection_ranges += col_statistics.hybrid_selection_ranges; + hybrid_selection_null_fallback_batches += + col_statistics.hybrid_selection_null_fallback_batches; decode_dict_time += col_statistics.decode_dict_time; decode_level_time += col_statistics.decode_level_time; decode_null_map_time += col_statistics.decode_null_map_time; diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index ab48e0f34b1330..d1e861f20d0211 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -114,6 +115,34 @@ class BaseDictDecoder : public Decoder { return Status::OK(); } + Status decode_selected_dictionary_indices(const ParquetSelection& selection, + std::vector* indices) override { + DORIS_CHECK(indices != nullptr); + _skip_indices.resize(selection.total_values); + const auto decoded = _index_batch_decoder->GetBatch( + _skip_indices.data(), cast_set(selection.total_values)); + if (UNLIKELY(decoded != selection.total_values)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t num_dictionary_values = dictionary_size(); + for (size_t row = 0; row < selection.total_values; ++row) { + if (UNLIKELY(_skip_indices[row] >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + _skip_indices[row], row, num_dictionary_values); + } + } + indices->resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + memcpy(indices->data() + output, _skip_indices.data() + range.first, + range.count * sizeof(uint32_t)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return Status::OK(); + } + protected: Status skip_values(size_t num_values) override { _skip_indices.resize(num_values); diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index 4358d3908114be..13d041b033f025 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -19,9 +19,11 @@ #include #include -#include -#include +#include +#include +#include +#include #include #include #include @@ -82,6 +84,26 @@ class DeltaBitPackDecoder final : public DeltaDecoder { sizeof(T)); } + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + _values.resize(selection.total_values); + uint32_t decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta values, decoded {}", + selection.total_values, decoded_count); + } + size_t output = 0; + for (const auto& range : selection.ranges) { + memmove(_values.data() + output, _values.data() + range.first, range.count * sizeof(T)); + output += range.count; + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(reinterpret_cast(_values.data()), output, + sizeof(T)); + } + Status decode(T* buffer, uint32_t num_values, uint32_t* out_num_values) { return _get_internal(buffer, num_values, out_num_values); } @@ -166,6 +188,28 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { return consumer.consume(_string_refs.data(), _string_refs.size()); } + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + _values.resize(selection.total_values); + int decoded_count = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), cast_set(selection.total_values), + &decoded_count)); + if (UNLIKELY(decoded_count != selection.total_values)) { + return Status::IOError("Expected {} Parquet delta-length values, decoded {}", + selection.total_values, decoded_count); + } + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + Status decode(Slice* buffer, int num_values, int* out_num_values) { return _get_internal(buffer, num_values, out_num_values); } @@ -223,6 +267,21 @@ class DeltaByteArrayDecoder : public DeltaDecoder { return consumer.consume(_string_refs.data(), _string_refs.size()); } + Status decode_selected_binary_values(const ParquetSelection& selection, + ParquetBinaryValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(selection.total_values)); + _string_refs.resize(selection.selected_values); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + _string_refs[output++] = StringRef(value.data, value.size); + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_string_refs.data(), _string_refs.size()); + } + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { RETURN_IF_ERROR(_decode_slices(num_values)); const size_t byte_size = num_values * static_cast(_type_length); @@ -238,6 +297,32 @@ class DeltaByteArrayDecoder : public DeltaDecoder { static_cast(_type_length)); } + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override { + RETURN_IF_ERROR(_decode_slices(selection.total_values)); + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.selected_values > + std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet delta-byte-array selection byte size overflows"); + } + _fixed_values.resize(selection.selected_values * value_width); + size_t output = 0; + for (const auto& range : selection.ranges) { + for (size_t row = 0; row < range.count; ++row) { + const auto& value = _values[range.first + row]; + if (UNLIKELY(value.size != value_width)) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + value.size, value_width); + } + memcpy(_fixed_values.data() + output * value_width, value.data, value_width); + ++output; + } + } + DORIS_CHECK_EQ(output, selection.selected_values); + return consumer.consume(_fixed_values.data(), output, value_width); + } + Status set_data(Slice* slice) override { _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(_bit_reader)); diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp index 463ef3db82a394..09f019c7dc0033 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -17,6 +17,9 @@ #include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" +#include +#include + namespace doris::format::parquet::native { Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, @@ -31,6 +34,35 @@ Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, return Status::OK(); } +Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(selection.total_values > std::numeric_limits::max() / value_width || + selection.selected_values > std::numeric_limits::max() / value_width)) { + return Status::IOError("Parquet plain selection byte size overflows"); + } + const size_t input_bytes = selection.total_values * value_width; + if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { + return Status::IOError("Out-of-bounds access in Parquet plain selection decoder"); + } + _selected_values.resize(selection.selected_values * value_width); + size_t output_offset = 0; + for (const auto& range : selection.ranges) { + const size_t range_bytes = range.count * value_width; + memcpy(_selected_values.data() + output_offset, + reinterpret_cast(_data->data) + _offset + range.first * value_width, + range_bytes); + output_offset += range_bytes; + } + DORIS_CHECK_EQ(output_offset, _selected_values.size()); + _offset += input_bytes; + if (_selected_values.empty()) { + return Status::OK(); + } + return consumer.consume(_selected_values.data(), selection.selected_values, value_width); +} + Status FixLengthPlainDecoder::skip_values(size_t num_values) { _offset += _type_length * num_values; if (UNLIKELY(_offset > _data->size)) { diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h index e0f3962ed0bde5..d08a9146ac5cc7 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -19,6 +19,8 @@ #include +#include + #include "common/status.h" #include "core/column/column_fixed_length_object.h" #include "core/data_type/data_type.h" @@ -38,7 +40,13 @@ class FixLengthPlainDecoder final : public Decoder { Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; + Status decode_selected_fixed_values(const ParquetSelection& selection, + ParquetFixedValueConsumer& consumer) override; + Status skip_values(size_t num_values) override; + +private: + std::vector _selected_values; }; } // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 849963a6666908..ebb92202e59aa4 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -557,6 +557,19 @@ int64_t NativeColumnReader::sync_native_profile() { COUNTER_UPDATE(_profile.materialization_time, stats.materialization_time - reported.materialization_time); } + if (_profile.hybrid_selection_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_batches, + stats.hybrid_selection_batches - reported.hybrid_selection_batches); + } + if (_profile.hybrid_selection_ranges != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_ranges, + stats.hybrid_selection_ranges - reported.hybrid_selection_ranges); + } + if (_profile.hybrid_selection_null_fallback_batches != nullptr) { + COUNTER_UPDATE(_profile.hybrid_selection_null_fallback_batches, + stats.hybrid_selection_null_fallback_batches - + reported.hybrid_selection_null_fallback_batches); + } if (_profile.page_index_read_calls != nullptr) { COUNTER_UPDATE(_profile.page_index_read_calls, stats.page_index_read_calls - reported.page_index_read_calls); diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 9c981c36c8467d..9d901270d5acb4 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -99,6 +99,22 @@ DorisUniqueBufferPtr make_byte_array_dictionary(const std::vector encode_plain_byte_arrays(const std::vector& values) { + size_t total_size = 0; + for (const auto& value : values) { + total_size += sizeof(uint32_t) + value.size(); + } + std::vector encoded(total_size); + size_t offset = 0; + for (const auto& value : values) { + encode_fixed32_le(encoded.data() + offset, static_cast(value.size())); + offset += sizeof(uint32_t); + memcpy(encoded.data() + offset, value.data(), value.size()); + offset += value.size(); + } + return encoded; +} + TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { int32_t dictionary_length = 0; auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); @@ -133,10 +149,81 @@ TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValida ParquetDecodeSource& source = decoder; EXPECT_TRUE(source.skip_values(1).is()); + // Sparse decode must validate filtered dictionary ids too. Predicate selection must never + // turn a corrupt page into a successful read merely because the bad row was not selected. + ASSERT_TRUE(decoder.set_data(&invalid_slice).ok()); + ParquetSelection filtered_corrupt {.total_values = 1, .selected_values = 0, .ranges = {}}; + EXPECT_TRUE(decoder.decode_selected_dictionary_indices(filtered_corrupt, &decoded_indices) + .is()); + Slice empty_indices; EXPECT_TRUE(decoder.set_data(&empty_indices).is()); } +TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { + const ParquetSelection selection { + .total_values = 7, + .selected_values = 3, + .ranges = {{.first = 1, .count = 2}, {.first = 6, .count = 1}}}; + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + std::vector integers {10, 11, 12, 13, 14, 15, 16, 17}; + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({11, 12, 16})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({17})); + + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded_strings = encode_plain_byte_arrays(strings); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + Slice string_slice(encoded_strings.data(), encoded_strings.size()); + ASSERT_TRUE(decoder->set_data(&string_slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 3); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "one"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "two"); + EXPECT_EQ(selected_strings.refs[2].to_string_view(), "six"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "seven"); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char booleans[] = {static_cast(0b10001101)}; + Slice boolean_slice(booleans, sizeof(booleans)); + ASSERT_TRUE(decoder->set_data(&boolean_slice).ok()); + CaptureFixedConsumer selected_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_booleans).ok()); + EXPECT_EQ(selected_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_boolean).ok()); + EXPECT_EQ(trailing_boolean.values(), std::vector({1})); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + char rle_booleans[] = {0x02, 0x00, 0x00, 0x00, 0x03, static_cast(0x8D)}; + Slice rle_boolean_slice(rle_booleans, sizeof(rle_booleans)); + ASSERT_TRUE(decoder->set_data(&rle_boolean_slice).ok()); + CaptureFixedConsumer selected_rle_booleans; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_rle_booleans).ok()); + EXPECT_EQ(selected_rle_booleans.values(), std::vector({0, 1, 0})); + CaptureFixedConsumer trailing_rle_boolean; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_rle_boolean).ok()); + EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); +} + TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { std::unique_ptr decoder; ASSERT_TRUE( @@ -227,6 +314,87 @@ TEST(ParquetV2NativeDecoderTest, DeltaEncodingsExposeValuesAfterSkip) { } } +TEST(ParquetV2NativeDecoderTest, SparseStatefulEncodingsBatchDecodeAndCompact) { + const ParquetSelection selection { + .total_values = 3, + .selected_values = 2, + .ranges = {{.first = 0, .count = 1}, {.first = 2, .count = 1}}}; + const std::vector integers {100, 101, 99, 1000}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto int_encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + int_encoder->Put(integers.data(), static_cast(integers.size())); + auto int_buffer = int_encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + Slice int_slice(int_buffer->data(), int_buffer->size()); + ASSERT_TRUE(decoder->set_data(&int_slice).ok()); + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), std::vector({100, 99})); + CaptureFixedConsumer trailing_integer; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_integer).ok()); + EXPECT_EQ(trailing_integer.values(), std::vector({1000})); + + const std::vector strings {"prefix-a", "prefix-b", "other", "other-tail"}; + std::vector<::parquet::ByteArray> byte_arrays; + byte_arrays.reserve(strings.size()); + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + encoding == ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + ? tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY + : tparquet::Encoding::DELTA_BYTE_ARRAY, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer selected_strings; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, selected_strings).ok()); + ASSERT_EQ(selected_strings.refs.size(), 2); + EXPECT_EQ(selected_strings.refs[0].to_string_view(), "prefix-a"); + EXPECT_EQ(selected_strings.refs[1].to_string_view(), "other"); + CaptureBinaryConsumer trailing_string; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing_string).ok()); + ASSERT_EQ(trailing_string.refs.size(), 1); + EXPECT_EQ(trailing_string.refs[0].to_string_view(), "other-tail"); + } + + const std::vector floats {1.0F, -2.5F, 3.25F, 9.5F}; + std::vector encoded_floats(floats.size() * sizeof(float)); + for (size_t row = 0; row < floats.size(); ++row) { + const auto* bytes = reinterpret_cast(&floats[row]); + for (size_t byte = 0; byte < sizeof(float); ++byte) { + encoded_floats[byte * floats.size() + row] = bytes[byte]; + } + } + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice float_slice(encoded_floats.data(), encoded_floats.size()); + ASSERT_TRUE(decoder->set_data(&float_slice).ok()); + CaptureFixedConsumer selected_floats; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_floats).ok()); + EXPECT_EQ(selected_floats.values(), std::vector({1.0F, 3.25F})); + CaptureFixedConsumer trailing_float; + ASSERT_TRUE(decoder->decode_fixed_values(1, trailing_float).ok()); + EXPECT_EQ(trailing_float.values(), std::vector({9.5F})); +} + TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { const std::vector values {1.0F, -2.5F, 3.25F}; std::vector encoded(values.size() * sizeof(float)); diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index 91d28a769deca4..169469d7d033ca 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -128,6 +128,14 @@ format-specific checklist when reviewing Parquet or ORC. - Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded values without allocating output; null runs must append defaults without advancing payload. +- For a filtered page fragment without definition-level NULLs, require one SerDe entry and one + batch-level selected-decode dispatch. Selection ranges belong to persistent reader scratch; + per-range virtual SerDe/decoder calls in the hot path are a review blocker. Fixed PLAIN should + bulk-gather spans, BYTE_ARRAY PLAIN should scan lengths once, dictionary decode should validate + every ID before gathering selected IDs, and stateful encodings should batch-decode/reconstruct and + compact. A NULL-interleaving fallback is acceptable only when it preserves logical output order + without a decoded intermediate column and is counted by + `HybridSelectionNullFallbackBatches`. - Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct without materializing an intermediate complex column. @@ -139,7 +147,8 @@ format-specific checklist when reviewing Parquet or ORC. deriving the synthetic child count, without constructing a discarded string/complex column. - `CountColumnReader` must use the native levels-only reader and must not decode payload or call Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, - value decode, SerDe materialization, filtered-value skips, and page fragmentation. + value decode, SerDe materialization, hybrid selection batches/ranges/NULL fallback, + filtered-value skips, and page fragmentation. ## Parquet Multi-Level Filtering diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 62bafc4fbd17ef..5ba0ca9ad621cd 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -369,10 +369,25 @@ selection, an empty selection, and an arbitrary fragmented selection use the sam Selection inputs are borrowed only for the duration of the decode call. For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes -definition-level runs, builds the four-way selection plan in a persistent action buffer whose -capacity survives adaptive batch changes, and dispatches its runs to the active encoding decoder. A -filtered non-null value must still advance the encoding state even when it is not copied. This is -the central invariant that prevents the next batch from decoding shifted values. +definition-level runs and builds the four-way selection plan in persistent scratch whose capacity +survives adaptive batch changes. When a filtered page fragment contains no definition-level NULL, +the plan is normalized to sorted physical ranges and enters `DataTypeSerDe` and the encoding +decoder once. This is the hybrid selection path: it keeps the dense direct-materialization path, +but moves sparse range traversal inside the concrete decoder instead of repeatedly constructing a +SerDe consumer for every selected run. + +The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk +copies; PLAIN BYTE_ARRAY scans every length once but records only selected references; dictionary +encoding decodes and validates the complete ID batch before gathering selected IDs; BOOLEAN, +DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected +values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate +Decoder/SerDe ownership boundary. A filtered non-null value always advances and validates encoding +state even when it is not copied, preventing the next batch from decoding shifted values. + +If selected NULL slots must be interleaved with values, v2 currently retains the four-run cursor +path so defaults and null-map bits are appended at the exact logical positions without a decoded +intermediate column. `HybridSelectionNullFallbackBatches` makes that conservative path visible; +it is a correctness boundary, not an Arrow fallback. ### 7.2 Page and Encoding Kernel @@ -383,13 +398,13 @@ that parsing step both feed the same level and value-decoder contracts. | Encoding family | Native responsibility | | --- | --- | -| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths | -| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, decode/skip IDs by selection, and reject invalid IDs | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; sparse fixed-width ranges use bulk gather and sparse strings scan lengths once | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, decode and validate the complete ID batch, then gather selected IDs | | RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | -| DELTA_BINARY_PACKED | Preserve block/mini-block state while selected and filtered values share one payload cursor | -| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, including skipped values | -| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value and binary scratch state | -| BYTE_STREAM_SPLIT | Reassemble primitive lanes and apply selection without an Arrow intermediate | +| DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | +| DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, then retain selected references only | +| DELTA_BYTE_ARRAY | Reconstruct prefix/suffix values with persistent previous-value state, then retain selected references only | +| BYTE_STREAM_SPLIT | Reassemble selected primitive lane ranges directly into one compact batch without an Arrow intermediate | Unsupported physical-type/encoding combinations return an explicit error. They never fall back to Arrow and never produce a plausible result through a decoder selected only by logical Doris type. @@ -752,7 +767,7 @@ flowchart TD | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | | Condition Cache | How many rows were skipped early after a cache hit? | -| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? | +| Native decode | How much time is spent in page parsing, decompression, levels, encoding, selection, conversion, string/fixed-binary materialization, and scratch growth? Compare `HybridSelectionBatches`, `HybridSelectionRanges`, and `HybridSelectionNullFallbackBatches` to distinguish batched sparse decode from NULL-interleaving fallback. | | Batch fragmentation | How does `TotalBatches` divide into adaptive probes, dense, selected, empty, page-crossing, and nested/fragmented batches? | | Index decisions | How often were statistics, dictionary, Bloom, ColumnIndex/OffsetIndex, and page skips attempted, accepted, conservatively rejected, or rejected as corrupt? | | Cache lifecycle | For footer/page/file/condition caches, what were request, hit, miss, bypass, admission/write, byte, wait, and underlying-I/O counts using v1-compatible meanings? | From 2a0d08242ec86353de77bc2677ab175e3ee922ba Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 03:08:14 +0800 Subject: [PATCH 09/34] [fix](be) Harden native parquet decoding and scan profiling Address the native Parquet review findings around malformed streams, page-cache identity and representation, nested-page accounting, offset-index ownership, and filtered decoding. Expose a consistent FileScannerV2 -> TableReader -> FileReader -> IO profile hierarchy with lifecycle and format/transport timers so slow file scans remain attributable. Tests: full BE unit suite (9664 definitions, 14 disabled, 0 failures); ./build.sh --be; clang-format/check-format. --- be/src/exec/scan/file_scanner_v2.cpp | 88 +++- be/src/exec/scan/file_scanner_v2.h | 9 + be/src/format/column_type_convert.cpp | 7 +- .../format/parquet/vparquet_file_metadata.cpp | 5 +- be/src/format/parquet/vparquet_reader.cpp | 3 +- .../delimited_text/delimited_text_reader.cpp | 13 +- .../delimited_text/delimited_text_reader.h | 1 + be/src/format_v2/jni/jni_table_reader.cpp | 11 +- be/src/format_v2/jni/jni_table_reader.h | 1 + be/src/format_v2/json/json_reader.cpp | 37 +- be/src/format_v2/json/json_reader.h | 8 + be/src/format_v2/native/native_reader.cpp | 40 +- be/src/format_v2/native/native_reader.h | 6 + .../format_v2/orc/orc_file_input_stream.cpp | 9 +- be/src/format_v2/orc/orc_reader.cpp | 36 +- be/src/format_v2/orc/orc_reader.h | 1 + .../parquet/parquet_file_context.cpp | 12 +- .../format_v2/parquet/parquet_file_context.h | 1 + be/src/format_v2/parquet/parquet_profile.cpp | 38 +- be/src/format_v2/parquet/parquet_profile.h | 2 + be/src/format_v2/parquet/parquet_reader.cpp | 8 + be/src/format_v2/parquet/parquet_scan.cpp | 4 +- be/src/format_v2/parquet/parquet_scan.h | 3 +- .../parquet/reader/count_column_reader.cpp | 5 +- .../parquet/reader/count_column_reader.h | 3 +- .../reader/native/bool_plain_decoder.cpp | 5 +- .../reader/native/bool_plain_decoder.h | 7 + .../reader/native/bool_rle_decoder.cpp | 13 +- .../parquet/reader/native/bool_rle_decoder.h | 7 +- .../reader/native/byte_array_plain_decoder.h | 7 + .../native/byte_stream_split_decoder.cpp | 7 +- .../reader/native/byte_stream_split_decoder.h | 16 + .../reader/native/column_chunk_reader.cpp | 78 ++- .../reader/native/column_chunk_reader.h | 22 +- .../parquet/reader/native/column_reader.cpp | 140 ++++-- .../parquet/reader/native/column_reader.h | 15 +- .../format_v2/parquet/reader/native/decoder.h | 30 +- .../reader/native/delta_bit_pack_decoder.cpp | 21 +- .../reader/native/delta_bit_pack_decoder.h | 136 ++++- .../reader/native/fix_length_plain_decoder.h | 7 + .../parquet/reader/native/level_decoder.cpp | 152 +++++- .../parquet/reader/native/level_decoder.h | 28 +- .../parquet/reader/native/level_reader.cpp | 13 +- .../parquet/reader/native/level_reader.h | 1 + .../parquet/reader/native/page_reader.cpp | 61 ++- .../parquet/reader/native/page_reader.h | 29 +- .../parquet/reader/native_column_reader.cpp | 60 ++- .../parquet/reader/native_column_reader.h | 12 +- ...eberg_position_delete_sys_table_reader.cpp | 1 + .../format_v2/table/remote_doris_reader.cpp | 44 +- be/src/format_v2/table/remote_doris_reader.h | 7 + be/src/format_v2/table_reader.cpp | 49 +- be/src/format_v2/table_reader.h | 53 +- be/src/io/cache/block_file_cache_profile.cpp | 23 +- be/src/io/cache/block_file_cache_profile.h | 5 +- be/src/io/fs/buffered_reader.cpp | 11 +- be/src/io/fs/buffered_reader.h | 8 +- be/src/io/fs/file_meta_cache.cpp | 11 + be/src/io/fs/file_meta_cache.h | 4 + be/src/io/fs/hdfs_file_reader.cpp | 6 +- be/src/io/fs/hdfs_file_reader.h | 1 + be/src/io/fs/s3_file_reader.cpp | 6 +- be/src/runtime/file_scan_profile.h | 55 +++ be/test/exec/scan/file_scanner_v2_test.cpp | 9 + .../file_reader/file_meta_cache_test.cpp | 9 +- .../format_v2/parquet/native_decoder_test.cpp | 464 ++++++++++++++++++ .../format_v2/parquet/parquet_reader_test.cpp | 68 +++ ...block_file_cache_profile_reporter_test.cpp | 24 + 68 files changed, 1808 insertions(+), 268 deletions(-) create mode 100644 be/src/runtime/file_scan_profile.h diff --git a/be/src/exec/scan/file_scanner_v2.cpp b/be/src/exec/scan/file_scanner_v2.cpp index f06e9fb9533774..f2dd05854c2362 100644 --- a/be/src/exec/scan/file_scanner_v2.cpp +++ b/be/src/exec/scan/file_scanner_v2.cpp @@ -65,6 +65,7 @@ #include "io/io_common.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "service/backend_options.h" #include "storage/id_manager.h" @@ -324,26 +325,42 @@ FileScannerV2::FileScannerV2(RuntimeState* state, FileScanLocalState* local_stat Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjuncts) { RETURN_IF_ERROR(Scanner::init(state, conjuncts)); - _get_block_timer = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileScannerV2GetBlockTime", 1); - _empty_file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "EmptyFileNum", TUnit::UNIT, 1); - _not_found_file_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "NotFoundFileNum", TUnit::UNIT, 1); - _file_counter = - ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), "FileNumber", TUnit::UNIT, 1); - _file_read_bytes_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadBytes", TUnit::BYTES, 1); - _file_read_calls_counter = ADD_COUNTER_WITH_LEVEL(_local_state->scanner_profile(), - "FileReadCalls", TUnit::UNIT, 1); + auto* profile = _local_state->scanner_profile(); + const auto hierarchy = file_scan_profile::ensure_hierarchy(profile); + _scanner_total_timer = hierarchy.scanner; + _io_timer = hierarchy.io; + _init_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2InitTime", + file_scan_profile::SCANNER, 1); + _open_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2OpenTime", + file_scan_profile::SCANNER, 1); + _get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetBlockTime", + file_scan_profile::SCANNER, 1); + _prepare_split_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2PrepareSplitTime", + file_scan_profile::SCANNER, 1); + _get_next_range_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2GetNextRangeTime", + file_scan_profile::SCANNER, 1); + _close_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileScannerV2CloseTime", + file_scan_profile::SCANNER, 1); + _empty_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "EmptyFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _not_found_file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NotFoundFileNum", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNumber", TUnit::UNIT, + file_scan_profile::SCANNER, 1); + _file_read_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadBytes", TUnit::BYTES, + file_scan_profile::IO, 1); + _file_read_calls_counter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileReadCalls", TUnit::UNIT, + file_scan_profile::IO, 1); _file_read_time_counter = - ADD_TIMER_WITH_LEVEL(_local_state->scanner_profile(), "FileReadTime", 1); - _adaptive_batch_predicted_rows_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchPredictedRows", TUnit::UNIT, 1); - _adaptive_batch_actual_bytes_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchActualBytes", TUnit::BYTES, 1); - _adaptive_batch_probe_count_counter = ADD_COUNTER_WITH_LEVEL( - _local_state->scanner_profile(), "AdaptiveBatchProbeCount", TUnit::UNIT, 1); + ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReadTime", file_scan_profile::IO, 1); + _adaptive_batch_predicted_rows_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchPredictedRows", TUnit::UNIT, file_scan_profile::SCANNER, 1); + _adaptive_batch_actual_bytes_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchActualBytes", TUnit::BYTES, file_scan_profile::SCANNER, 1); + _adaptive_batch_probe_count_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "AdaptiveBatchProbeCount", TUnit::UNIT, file_scan_profile::SCANNER, 1); + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_init_timer); _file_cache_statistics = std::make_unique(); _file_reader_stats = std::make_unique(); RETURN_IF_ERROR(_init_io_ctx()); @@ -354,6 +371,8 @@ Status FileScannerV2::init(RuntimeState* state, const VExprContextSPtrs& conjunc } Status FileScannerV2::_open_impl(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_open_timer); RETURN_IF_CANCELLED(state); RETURN_IF_ERROR(Scanner::_open_impl(state)); RETURN_IF_ERROR(_get_next_scan_range(&_first_scan_range)); @@ -367,6 +386,7 @@ Status FileScannerV2::_open_impl(RuntimeState* state) { } Status FileScannerV2::_get_next_scan_range(bool* has_next) { + SCOPED_TIMER(_get_next_range_timer); DORIS_CHECK(has_next != nullptr); RETURN_IF_ERROR(_split_source->get_next(has_next, &_current_range)); if (*has_next) { @@ -376,6 +396,8 @@ Status FileScannerV2::_get_next_scan_range(bool* has_next) { } Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* eof) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_get_block_timer); while (true) { RETURN_IF_CANCELLED(state); if (!_has_prepared_split) { @@ -386,7 +408,6 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } { - SCOPED_TIMER(_get_block_timer); if (_should_run_adaptive_batch_size()) { _table_reader->set_batch_size(_predict_reader_batch_rows()); } @@ -428,6 +449,7 @@ Status FileScannerV2::_get_block_impl(RuntimeState* state, Block* block, bool* e } Status FileScannerV2::_prepare_next_split(bool* eos) { + SCOPED_TIMER(_prepare_split_timer); while (true) { bool has_next = _first_scan_range; if (!_first_scan_range) { @@ -906,6 +928,8 @@ void FileScannerV2::_update_adaptive_batch_size(const Block& block) { } Status FileScannerV2::close(RuntimeState* state) { + SCOPED_TIMER(_scanner_total_timer); + SCOPED_TIMER(_close_timer); if (!_try_close()) { return Status::OK(); } @@ -1045,14 +1069,31 @@ void FileScannerV2::_collect_profile_before_close() { Scanner::_collect_profile_before_close(); if (config::enable_file_cache && _state->query_options().enable_file_cache && _profile != nullptr) { - _report_file_cache_profile(_profile, *_file_cache_statistics); + auto file_cache_delta = io::diff_file_cache_statistics(*_file_cache_statistics, + _reported_file_cache_statistics); + // Profile collection can run more than once. Keep additive fields incremental while + // publishing high-water gauges and peer identities from the latest complete snapshot. + file_cache_delta.remote_only_on_miss_triggered = + _file_cache_statistics->remote_only_on_miss_triggered; + file_cache_delta.remote_only_on_miss_threshold_bytes = + _file_cache_statistics->remote_only_on_miss_threshold_bytes; + file_cache_delta.peer_hosts = _file_cache_statistics->peer_hosts; + _report_file_cache_profile(_profile, file_cache_delta); _state->get_query_ctx()->resource_ctx()->io_context()->update_bytes_write_into_cache( - _file_cache_statistics->bytes_write_into_cache); + file_cache_delta.bytes_write_into_cache); + _reported_file_cache_statistics = *_file_cache_statistics; } if (_file_reader_stats != nullptr) { COUNTER_SET(_file_read_bytes_counter, cast_set(_file_reader_stats->read_bytes)); COUNTER_SET(_file_read_calls_counter, cast_set(_file_reader_stats->read_calls)); COUNTER_SET(_file_read_time_counter, cast_set(_file_reader_stats->read_time_ns)); + const auto read_time = cast_set(_file_reader_stats->read_time_ns); + DORIS_CHECK(read_time >= _reported_io_read_time); + // Some transports (for example Arrow Flight) record directly into IO, while filesystem + // reads arrive through FileReaderStats. Add only the new traced delta so both paths remain + // visible without double counting repeated profile publication. + COUNTER_UPDATE(_io_timer, read_time - _reported_io_read_time); + _reported_io_read_time = read_time; } // Query profiles can be collected before Scanner::close() runs. Publish condition-cache // counters here as well, using deltas so this method and close() cannot double count. @@ -1061,7 +1102,8 @@ void FileScannerV2::_collect_profile_before_close() { void FileScannerV2::_report_file_cache_profile( RuntimeProfile* profile, const io::FileCacheStatistics& file_cache_statistics) { - io::FileCacheProfileReporter cache_profile(profile); + file_scan_profile::ensure_hierarchy(profile); + io::FileCacheProfileReporter cache_profile(profile, file_scan_profile::IO); cache_profile.update(&file_cache_statistics); } diff --git a/be/src/exec/scan/file_scanner_v2.h b/be/src/exec/scan/file_scanner_v2.h index 0cd03030b56851..1d2a62f8f9af1a 100644 --- a/be/src/exec/scan/file_scanner_v2.h +++ b/be/src/exec/scan/file_scanner_v2.h @@ -185,12 +185,20 @@ class FileScannerV2 final : public Scanner { std::unordered_map _partition_slot_descs; std::unique_ptr _file_cache_statistics; + io::FileCacheStatistics _reported_file_cache_statistics; std::unique_ptr _file_reader_stats; std::shared_ptr _io_ctx; ShardedKVCache* _kv_cache = nullptr; + RuntimeProfile::Counter* _scanner_total_timer = nullptr; + RuntimeProfile::Counter* _init_timer = nullptr; + RuntimeProfile::Counter* _open_timer = nullptr; RuntimeProfile::Counter* _get_block_timer = nullptr; RuntimeProfile::Counter* _empty_file_counter = nullptr; + RuntimeProfile::Counter* _prepare_split_timer = nullptr; + RuntimeProfile::Counter* _get_next_range_timer = nullptr; + RuntimeProfile::Counter* _close_timer = nullptr; + RuntimeProfile::Counter* _io_timer = nullptr; RuntimeProfile::Counter* _not_found_file_counter = nullptr; RuntimeProfile::Counter* _file_counter = nullptr; RuntimeProfile::Counter* _file_read_bytes_counter = nullptr; @@ -207,6 +215,7 @@ class FileScannerV2 final : public Scanner { int64_t _last_read_rows = 0; int64_t _last_bytes_read_from_local = 0; int64_t _last_bytes_read_from_remote = 0; + int64_t _reported_io_read_time = 0; }; } // namespace doris diff --git a/be/src/format/column_type_convert.cpp b/be/src/format/column_type_convert.cpp index 08fe5c8a4ce794..414691d386d201 100644 --- a/be/src/format/column_type_convert.cpp +++ b/be/src/format/column_type_convert.cpp @@ -109,7 +109,12 @@ ColumnPtr ColumnTypeConverter::get_column(const DataTypePtr& src_type, ColumnPtr } if (!_cached_src_column) { - _cached_src_type = dst_type->is_nullable() + // Projection metadata can be non-nullable while the actual output block keeps a nullable + // wrapper (for example an Iceberg equality-delete key). Mirror the physical destination + // column so decoded null levels always have a temporary null map to propagate. + const bool destination_is_nullable = + dst_type->is_nullable() || is_column_nullable(*dst_column); + _cached_src_type = destination_is_nullable ? get_data_type_with_default_argument(make_nullable(src_type)) : get_data_type_with_default_argument(remove_nullable(src_type)); _cached_src_column = _cached_src_type->create_column(); diff --git a/be/src/format/parquet/vparquet_file_metadata.cpp b/be/src/format/parquet/vparquet_file_metadata.cpp index e2886ef624ec9e..abca40d7f6a6e5 100644 --- a/be/src/format/parquet/vparquet_file_metadata.cpp +++ b/be/src/format/parquet/vparquet_file_metadata.cpp @@ -44,7 +44,10 @@ Status FileMetaData::init_schema(const bool enable_mapping_varbinary, } _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); - return _schema.parse_from_thrift(_metadata.schema); + RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); + // Cached metadata is immutable after publication; assign native field IDs before insertion. + _schema.assign_ids(); + return Status::OK(); } const tparquet::FileMetaData& FileMetaData::to_thrift() const { diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 26caf7faac66f2..7a8b8f70de072a 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -368,7 +368,8 @@ Status ParquetReader::_open_file() { _reader_statistics.file_footer_read_calls += 1; } else { const auto& file_meta_cache_key = - FileMetaCache::get_key(_tracing_file_reader, _file_description); + FileMetaCache::get_key(_tracing_file_reader, _file_description, + enable_mapping_varbinary, enable_mapping_timestamp_tz); if (!_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, &meta_size, _io_ctx, enable_mapping_varbinary, diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp index f19d12c75714b9..273bb0a0785a75 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -38,6 +38,7 @@ #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/string_util.h" @@ -180,7 +181,9 @@ void DelimitedTextReader::_init_profile() { return; } - ADD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, 1); + file_scan_profile::ensure_hierarchy(_profile); + _text_profile.total_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, DELIMITED_TEXT_PROFILE, + file_scan_profile::FILE_READER, 1); _text_profile.open_file_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OpenFileTime", DELIMITED_TEXT_PROFILE, 1); _text_profile.create_line_reader_time = @@ -200,7 +203,7 @@ void DelimitedTextReader::_init_profile() { _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + _profile, "RowsFilteredByConjunct", TUnit::UNIT, file_scan_profile::FILE_READER, 1); _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -215,6 +218,7 @@ void DelimitedTextReader::_init_profile() { Status DelimitedTextReader::init(RuntimeState* state) { _init_profile(); + SCOPED_TIMER(_text_profile.total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("{} v2 reader requires scan params", _reader_name); @@ -275,6 +279,7 @@ Status DelimitedTextReader::init(RuntimeState* state) { } Status DelimitedTextReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_text_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("{} v2 file_schema is null", _reader_name); } @@ -288,6 +293,7 @@ std::unique_ptr DelimitedTextReader::create_column_mapper( } Status DelimitedTextReader::open(std::shared_ptr request) { + SCOPED_TIMER(_text_profile.total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -307,6 +313,7 @@ Status DelimitedTextReader::open(std::shared_ptr request) { } Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -358,6 +365,7 @@ Status DelimitedTextReader::get_block(Block* file_block, size_t* rows, bool* eof Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& request, FileAggregateResult* result) { + SCOPED_TIMER(_text_profile.total_time); DORIS_CHECK(result != nullptr); if (request.agg_type != TPushAggOp::type::COUNT) { return Status::NotSupported("{} v2 reader only supports COUNT aggregate pushdown", @@ -396,6 +404,7 @@ Status DelimitedTextReader::get_aggregate_result(const FileAggregateRequest& req } Status DelimitedTextReader::close() { + SCOPED_TIMER(_text_profile.total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.h b/be/src/format_v2/delimited_text/delimited_text_reader.h index daea3bc90942d8..dff27980c9cd12 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.h +++ b/be/src/format_v2/delimited_text/delimited_text_reader.h @@ -59,6 +59,7 @@ class DelimitedTextReader : public FileReader { protected: struct DelimitedTextProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* open_file_time = nullptr; RuntimeProfile::Counter* create_line_reader_time = nullptr; RuntimeProfile::Counter* read_line_time = nullptr; diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index 3cb105193b56ae..dd858e4a20de8e 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -24,6 +24,7 @@ #include "core/block/block.h" #include "exprs/vexpr_context.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/string_util.h" @@ -32,10 +33,12 @@ namespace doris::format { Status JniTableReader::init(TableReadOptions&& options) { RETURN_IF_ERROR(TableReader::init(std::move(options))); _init_profile(); + SCOPED_TIMER(_connector_total_time); return Status::OK(); } Status JniTableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_connector_total_time); // EOF belongs to the previous split. Keep it set after closing that split so repeated reads // are idempotent, and clear it only when a new split is explicitly prepared. _eof = false; @@ -63,6 +66,9 @@ Status JniTableReader::prepare_split(const SplitReadOptions& options) { } Status JniTableReader::get_block(Block* output_block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.exec_timer); + SCOPED_TIMER(_connector_total_time); DORIS_CHECK(output_block != nullptr); DORIS_CHECK(eos != nullptr); DORIS_CHECK(output_block->columns() == _projected_columns.size()); @@ -342,6 +348,7 @@ void JniTableReader::_publish_split_profile(JNIEnv* env) { } Status JniTableReader::close() { + SCOPED_TIMER(_connector_total_time); if (_closed) { return Status::OK(); } @@ -535,7 +542,9 @@ void JniTableReader::_init_profile() { return; } const auto connector_name = _connector_name(); - ADD_TIMER(_scanner_profile, connector_name); + file_scan_profile::ensure_hierarchy(_scanner_profile); + _connector_total_time = + ADD_CHILD_TIMER(_scanner_profile, connector_name, file_scan_profile::TABLE_READER); _open_scanner_time = ADD_CHILD_TIMER(_scanner_profile, "OpenScannerTime", connector_name); _java_scan_time = ADD_CHILD_TIMER(_scanner_profile, "JavaScanTime", connector_name); _java_append_data_time = diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index 24f84ca8d62756..4e3f4934f26334 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -110,6 +110,7 @@ class JniTableReader : public TableReader { bool _eof = false; bool _split_profile_published = false; + RuntimeProfile::Counter* _connector_total_time = nullptr; RuntimeProfile::Counter* _open_scanner_time = nullptr; RuntimeProfile::Counter* _java_scan_time = nullptr; RuntimeProfile::Counter* _java_append_data_time = nullptr; diff --git a/be/src/format_v2/json/json_reader.cpp b/be/src/format_v2/json/json_reader.cpp index 313115e82ce744..1f42a2cd2d5e94 100644 --- a/be/src/format_v2/json/json_reader.cpp +++ b/be/src/format_v2/json/json_reader.cpp @@ -47,6 +47,7 @@ #include "io/fs/stream_load_pipe.h" #include "io/fs/tracing_file_reader.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/decompressor.h" #include "util/slice.h" @@ -174,7 +175,26 @@ JsonReader::~JsonReader() { static_cast(close()); } +void JsonReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* json_profile = "JsonReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, json_profile, file_scan_profile::FILE_READER, 1); + _open_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonOpenTime", json_profile, 1); + _read_document_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonReadDocumentTime", json_profile, 1); + _parse_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonParseTime", json_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonMaterializeTime", json_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "JsonFilterTime", json_profile, 1); +} + Status JsonReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_scan_params == nullptr) { return Status::InvalidArgument("JSON v2 reader requires scan params"); @@ -230,6 +250,7 @@ Status JsonReader::init(RuntimeState* state) { } Status JsonReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("JSON v2 file_schema is null"); } @@ -243,6 +264,8 @@ std::unique_ptr JsonReader::create_column_mapper( } Status JsonReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); RETURN_IF_ERROR(_build_requested_columns(*_request, &_requested_columns)); @@ -269,6 +292,7 @@ Status JsonReader::open(std::shared_ptr request) { } Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -297,7 +321,10 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { bool is_empty_row = false; Status st = Status::OK(); try { - st = _parse_next_json(&size, &_reader_eof); + { + SCOPED_TIMER(_parse_time); + st = _parse_next_json(&size, &_reader_eof); + } if (st.ok() && !_reader_eof) { if (size == 0) { is_empty_row = true; @@ -306,6 +333,7 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { } } if (st.ok() && !_reader_eof && !is_empty_row) { + SCOPED_TIMER(_materialize_time); st = _append_rows_from_current_value(file_block, &is_empty_row, &_reader_eof); } } catch (simdjson::simdjson_error& e) { @@ -324,13 +352,17 @@ Status JsonReader::get_block(Block* file_block, size_t* rows, bool* eof) { *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } *eof = _reader_eof && *rows == 0; _eof = *eof; return Status::OK(); } Status JsonReader::close() { + SCOPED_TIMER(_total_time); if (_line_reader != nullptr) { _line_reader->close(); _line_reader.reset(); @@ -489,6 +521,7 @@ Status JsonReader::_parse_jsonpath_and_json_root() { } Status JsonReader::_read_one_document(size_t* size, bool* eof) { + SCOPED_TIMER(_read_document_time); DORIS_CHECK(size != nullptr); DORIS_CHECK(eof != nullptr); *size = 0; diff --git a/be/src/format_v2/json/json_reader.h b/be/src/format_v2/json/json_reader.h index 52cdfad6728d64..de3c084e681f23 100644 --- a/be/src/format_v2/json/json_reader.h +++ b/be/src/format_v2/json/json_reader.h @@ -75,6 +75,7 @@ class JsonReader final : public FileReader { Status close() override; private: + void _init_profile() override; // A requested column keeps both identities: // - `source_index`: index in FE file slots, used for jsonpaths and SerDe lookup. // - `block_position`: index in the caller's output block, used for materialization. @@ -137,6 +138,13 @@ class JsonReader final : public FileReader { std::unordered_map _slot_name_to_index; std::vector _previous_positions; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_time = nullptr; + RuntimeProfile::Counter* _read_document_time = nullptr; + RuntimeProfile::Counter* _parse_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; + io::FileReaderSPtr _physical_file_reader; std::unique_ptr _decompressor; std::unique_ptr _line_reader; diff --git a/be/src/format_v2/native/native_reader.cpp b/be/src/format_v2/native/native_reader.cpp index 5d0984084a6d41..3d5429e0a9a262 100644 --- a/be/src/format_v2/native/native_reader.cpp +++ b/be/src/format_v2/native/native_reader.cpp @@ -29,6 +29,7 @@ #include "format_v2/materialized_reader_util.h" #include "io/file_factory.h" #include "io/fs/tracing_file_reader.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/slice.h" @@ -54,7 +55,26 @@ NativeReader::~NativeReader() { static_cast(close()); } +void NativeReader::_init_profile() { + if (_profile == nullptr) { + return; + } + file_scan_profile::ensure_hierarchy(_profile); + static const char* native_profile = "NativeReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, native_profile, file_scan_profile::FILE_READER, 1); + _read_block_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeReadBlockTime", native_profile, 1); + _deserialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeDeserializeTime", native_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeMaterializeTime", native_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "NativeFilterTime", native_profile, 1); +} + Status NativeReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); _runtime_state = state; if (_file_description == nullptr) { return Status::InvalidArgument("Native v2 reader requires file description"); @@ -65,6 +85,7 @@ Status NativeReader::init(RuntimeState* state) { } Status NativeReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); if (file_schema == nullptr) { return Status::InvalidArgument("Native v2 file_schema is null"); } @@ -79,6 +100,7 @@ std::unique_ptr NativeReader::create_column_mapper( } Status NativeReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); DORIS_CHECK(_request != nullptr); _first_block_consumed = false; @@ -88,6 +110,7 @@ Status NativeReader::open(std::shared_ptr request) { } Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -108,6 +131,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { if (_first_block_loaded && !_first_block_consumed) { buffer = _first_block_buffer; } else { + SCOPED_TIMER(_read_block_time); RETURN_IF_ERROR(_read_next_pblock(&buffer, &local_eof)); } @@ -131,11 +155,20 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { Block source_block; size_t uncompressed_bytes = 0; int64_t decompress_time = 0; - RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); - RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + { + SCOPED_TIMER(_deserialize_time); + RETURN_IF_ERROR(source_block.deserialize(pblock, &uncompressed_bytes, &decompress_time)); + } + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_requested_columns(source_block, file_block)); + } *rows = file_block->rows(); _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR(_apply_filters(file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR(_apply_filters(file_block, rows)); + } if (_first_block_loaded && !_first_block_consumed) { _first_block_consumed = true; @@ -149,6 +182,7 @@ Status NativeReader::get_block(Block* file_block, size_t* rows, bool* eof) { } Status NativeReader::close() { + SCOPED_TIMER(_total_time); _file_reader.reset(); _tracing_file_reader.reset(); _request.reset(); diff --git a/be/src/format_v2/native/native_reader.h b/be/src/format_v2/native/native_reader.h index 3719a6afd6c4f5..15a52fe6f8bc87 100644 --- a/be/src/format_v2/native/native_reader.h +++ b/be/src/format_v2/native/native_reader.h @@ -49,6 +49,7 @@ class NativeReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _validate_and_consume_header(); Status _ensure_schema_loaded() const; Status _read_next_pblock(std::string* buffer, bool* eof) const; @@ -65,6 +66,11 @@ class NativeReader final : public FileReader { mutable std::string _first_block_buffer; mutable bool _first_block_loaded = false; mutable bool _first_block_consumed = false; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _read_block_time = nullptr; + RuntimeProfile::Counter* _deserialize_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; }; } // namespace doris::format::native diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index a5144d7b2017b0..303a65c4947bae 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -27,6 +27,7 @@ #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "orc/Exceptions.hh" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "util/slice.h" @@ -54,8 +55,10 @@ class OrcMergedRangeFileReader final : public io::FileReader { _size(_file_reader->size()) { _statistics.apply_bytes += _range.end_offset - _range.start_offset; if (_profile != nullptr) { - const char* profile_name = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, profile_name, 1); + const char* profile_name = "OrcMergedSmallIO"; + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, @@ -113,6 +116,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { if (_profile == nullptr) { return; } + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -165,6 +169,7 @@ class OrcMergedRangeFileReader final : public io::FileReader { OrcMergedRangeStatistics _statistics; RuntimeProfile::Counter* _copy_time = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; RuntimeProfile::Counter* _merged_io = nullptr; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 9a2aeb538977dc..076b79a6885d00 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -76,6 +76,7 @@ #include "format_v2/timestamp_statistics.h" #include "io/fs/file_reader.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/segment/condition_cache.h" @@ -776,7 +777,9 @@ void OrcReader::_init_profile() { } static const char* orc_profile = "OrcReader"; - ADD_TIMER_WITH_LEVEL(_profile, orc_profile, 1); + file_scan_profile::ensure_hierarchy(_profile); + _orc_profile.total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, orc_profile, file_scan_profile::FILE_READER, 1); _orc_profile.reader_call = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReaderCall", TUnit::UNIT, orc_profile, 1); _orc_profile.reader_inclusive_latency_us = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -803,20 +806,20 @@ void OrcReader::_init_profile() { _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); _orc_profile.read_row_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsFiltered", - TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowGroupsFiltered", TUnit::UNIT, file_scan_profile::FILE_READER, 1); _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); - _orc_profile.read_row_groups = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RowGroupsReadNum", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredRowsByGroup", - TUnit::UNIT, orc_profile, 1); + _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "RowGroupsReadNum", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "FilteredRowsByGroup", TUnit::UNIT, file_scan_profile::FILE_READER, 1); _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); - _orc_profile.filtered_bytes = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FilteredBytes", TUnit::BYTES, orc_profile, 1); - _orc_profile.open_file_num = - ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, orc_profile, 1); + _profile, "FilteredRowsByLazyRead", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + _profile, "FilteredBytes", TUnit::BYTES, file_scan_profile::FILE_READER, 1); + _orc_profile.open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, + file_scan_profile::FILE_READER, 1); } void OrcReader::_collect_profile() const { @@ -874,6 +877,8 @@ format::ColumnDefinition OrcReader::row_position_column_definition() { } Status OrcReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_orc_profile.total_time); RETURN_IF_ERROR(format::FileReader::init(state)); _state = std::make_unique(); TimezoneUtils::find_cctz_time_zone(_state->timezone, _state->timezone_obj); @@ -1162,6 +1167,7 @@ Status OrcReader::_fill_map_schema_children(const ::orc::Type& type, } Status OrcReader::get_schema(std::vector* const file_schema) const { + SCOPED_TIMER(_orc_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } @@ -1195,6 +1201,7 @@ std::unique_ptr OrcReader::create_column_mapper( } Status OrcReader::open(std::shared_ptr request) { + SCOPED_TIMER(_orc_profile.total_time); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); } @@ -1877,6 +1884,7 @@ Status OrcReader::_decode_column(const ::orc::Type& file_type, const ::orc::Type } Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -2000,6 +2008,7 @@ Status OrcReader::get_block(Block* file_block, size_t* rows, bool* eof) { // NOLINTNEXTLINE(readability-function-size) Status OrcReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_orc_profile.total_time); DORIS_CHECK(result != nullptr); if (_state == nullptr || _state->reader == nullptr || _state->root_type == nullptr) { return Status::Uninitialized("OrcReader is not open"); @@ -2455,6 +2464,7 @@ void OrcReader::_filter_requested_columns(Block* file_block, const IColumn::Filt } Status OrcReader::close() { + SCOPED_TIMER(_orc_profile.total_time); _collect_profile(); if (_state != nullptr) { _state = std::make_unique(); diff --git a/be/src/format_v2/orc/orc_reader.h b/be/src/format_v2/orc/orc_reader.h index adba20fbce0b10..a67dc5ad7f5e3e 100644 --- a/be/src/format_v2/orc/orc_reader.h +++ b/be/src/format_v2/orc/orc_reader.h @@ -72,6 +72,7 @@ class OrcReader final : public format::FileReader { private: struct OrcProfile { + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* reader_call = nullptr; // ReaderCall RuntimeProfile::Counter* reader_inclusive_latency_us = nullptr; // ReaderInclusiveLatencyUs RuntimeProfile::Counter* decompression_call = nullptr; // DecompressionCall diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 27cc579ecb2975..45b9cc4c5af7cd 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -585,7 +585,9 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont // Arrow metadata is opened: native readers can reuse a footer produced by a v1 scan (and vice // versa), and a cache miss performs one bounded tail read through the same Doris FileReader. auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); - const auto meta_cache_key = FileMetaCache::get_key(native_file, file_description); + const auto meta_cache_key = + FileMetaCache::get_key(native_file, file_description, /*enable_mapping_varbinary=*/true, + enable_mapping_timestamp_tz); size_t native_footer_size = 0; if (meta_cache != nullptr && meta_cache->enabled() && meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { @@ -605,10 +607,11 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont } } DORIS_CHECK(native_metadata != nullptr); - const_cast(native_metadata->schema()).assign_ids(); auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); + // Native and Arrow readers must use the same FileDescription-derived immutable identity. + native_page_cache_file_key = page_cache_file_key; arrow_file = std::make_shared( input_file_reader, io_ctx, enable_page_cache, std::move(page_cache_file_key)); try { @@ -670,7 +673,9 @@ Status ParquetFileContext::load_native_offset_indexes( } for (const int leaf_column_id : leaf_column_ids) { auto arrow_index = row_group_reader->GetOffsetIndex(leaf_column_id); - if (arrow_index == nullptr) { + if (arrow_index == nullptr || arrow_index->page_locations().empty()) { + // An empty optional index is equivalent to no index. Publishing it would select + // the indexed PageReader even though there is no first page to dereference. continue; } tparquet::OffsetIndex native_index; @@ -776,6 +781,7 @@ Status ParquetFileContext::close() { native_file.reset(); native_io_ctx = nullptr; native_page_cache_enabled = false; + native_page_cache_file_key.clear(); return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 72eacd9a9adeb0..c1457547003a1c 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -162,6 +162,7 @@ struct ParquetFileContext { int64_t native_footer_read_calls = 0; int64_t native_footer_cache_hits = 0; bool native_page_cache_enabled = false; + std::string native_page_cache_file_key; std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 597f86010a2798..2951fccf2eb8a2 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/parquet_statistics.h" +#include "runtime/file_scan_profile.h" namespace doris::format::parquet { @@ -26,13 +27,17 @@ void ParquetProfile::init(RuntimeProfile* profile) { return; } + file_scan_profile::ensure_hierarchy(profile); static const char* parquet_profile = "ParquetReader"; - ADD_TIMER_WITH_LEVEL(profile, parquet_profile, 1); + total_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); + // These counters are format-independent and can be reused when one scanner switches between + // Parquet and ORC splits; keep their single flat counter identity under FileReader. filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, - parquet_profile, 1); + file_scan_profile::FILE_READER, 1); filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, parquet_profile, 1); + profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, file_scan_profile::FILE_READER, 1); filtered_row_groups_by_dictionary = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "RowGroupsFilteredByDictionary", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_bloom_filter = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -40,13 +45,13 @@ void ParquetProfile::init(RuntimeProfile* profile) { filtered_row_groups_by_page_index = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "RowGroupsFilteredByPageIndex", TUnit::UNIT, parquet_profile, 1); to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsReadNum", TUnit::UNIT, - parquet_profile, 1); + file_scan_profile::FILE_READER, 1); total_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsTotalNum", TUnit::UNIT, parquet_profile, 1); selected_row_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRowRanges", TUnit::UNIT, parquet_profile, 1); filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByGroup", TUnit::UNIT, - parquet_profile, 1); + file_scan_profile::FILE_READER, 1); filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByPage", TUnit::UNIT, parquet_profile, 1); pages_skipped_by_data_page_filter = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -55,8 +60,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); selected_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); - rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", - TUnit::UNIT, parquet_profile, 1); + rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "RowsFilteredByConjunct", TUnit::UNIT, file_scan_profile::FILE_READER, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); dense_batches = @@ -93,10 +98,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); nested_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); - lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", - TUnit::UNIT, parquet_profile, 1); + lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "FilteredRowsByLazyRead", TUnit::UNIT, file_scan_profile::FILE_READER, 1); filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, - parquet_profile, 1); + file_scan_profile::FILE_READER, 1); raw_rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); @@ -104,9 +109,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); file_reader_create_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); - open_file_num = - ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); - page_index_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, 1); + open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, + file_scan_profile::FILE_READER, 1); + page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, + parquet_profile, 1); page_index_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PageIndexFilterTime", parquet_profile, 1); read_page_index_time = @@ -121,8 +127,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); row_group_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "RowGroupFilterTime", parquet_profile, 1); - file_footer_read_calls = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", TUnit::UNIT, 1); - file_footer_hit_cache = ADD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, 1); + file_footer_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterReadCalls", + TUnit::UNIT, parquet_profile, 1); + file_footer_hit_cache = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileFooterHitCache", TUnit::UNIT, + parquet_profile, 1); decompress_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DecompressTime", parquet_profile, 1); decompress_cnt = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "DecompressCount", TUnit::UNIT, parquet_profile, 1); diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index a1a79973f446e3..1b11001426b560 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -112,6 +112,8 @@ struct ParquetProfile { ParquetColumnReaderProfile column_reader_profile() const; ParquetScanProfile scan_profile() const; + RuntimeProfile::Counter* total_time = nullptr; + RuntimeProfile::Counter* filtered_row_groups = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_min_max = nullptr; RuntimeProfile::Counter* filtered_row_groups_by_dictionary = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index c3453e8749e548..9bc330fa0d34f2 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -393,6 +393,8 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p ParquetReader::~ParquetReader() = default; Status ParquetReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_parquet_profile.total_time); if (_io_ctx != nullptr && _io_ctx->should_stop) { return Status::EndOfFile("stop"); } @@ -451,6 +453,7 @@ void ParquetReader::set_batch_size(size_t batch_size) { } Status ParquetReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_parquet_profile.total_time); if (file_schema == nullptr) { return Status::InvalidArgument("file_schema is null"); } @@ -478,6 +481,7 @@ std::unique_ptr ParquetReader::create_column_mapper( } Status ParquetReader::open(std::shared_ptr request) { + SCOPED_TIMER(_parquet_profile.total_time); if (_state == nullptr || _state->file_context.metadata == nullptr || _state->file_context.schema == nullptr) { return Status::Uninitialized("ParquetReader is not open"); @@ -553,6 +557,7 @@ Status ParquetReader::open(std::shared_ptr request) { } Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_parquet_profile.total_time); if (_state == nullptr || _state->file_context.file_reader == nullptr || _state->file_context.schema == nullptr) { return Status::Uninitialized("ParquetReader is not open"); @@ -653,6 +658,7 @@ int64_t ParquetReader::get_total_rows() const { Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& request, format::FileAggregateResult* result) { + SCOPED_TIMER(_parquet_profile.total_time); DORIS_CHECK(result != nullptr); if (_state == nullptr || _state->file_context.metadata == nullptr || _state->file_context.schema == nullptr) { @@ -716,6 +722,7 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r row_group_plan.row_group_id, root_schema, &count_projection, _state->file_context.native_io_ctx, _state->file_context.native_page_cache_enabled, + _state->file_context.native_page_cache_file_key, _parquet_profile.scan_profile().column_reader_profile, &shape_reader)); DORIS_CHECK(shape_reader != nullptr); @@ -797,6 +804,7 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r } Status ParquetReader::close() { + SCOPED_TIMER(_parquet_profile.total_time); if (_state != nullptr) { _state->scheduler.close(); _sync_page_cache_profile(); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 8790973566cdf0..4c0d06eef5b0a8 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -806,6 +806,7 @@ Status ParquetScanScheduler::open_next_row_group( *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, _current_dictionary_filters.contains(local_id), _scan_profile.column_reader_profile, &column_reader)); _current_predicate_columns[local_id] = std::move(column_reader); @@ -842,7 +843,8 @@ Status ParquetScanScheduler::open_next_row_group( *column_schema, &col, file_context.native_data_file(), file_context.native_metadata, row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, - false, _scan_profile.column_reader_profile, &column_reader)); + file_context.native_page_cache_file_key, false, _scan_profile.column_reader_profile, + &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } if (!_current_merge_range_active && request.conjuncts.empty() && diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index da0a47aed3f443..f1b2d8b1c7b802 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -190,6 +190,8 @@ class ParquetScanScheduler { size_t _next_row_group_plan_idx = 0; // index of the next row group to process bool _has_current_row_group = false; + // Readers retain pointers into this immutable row-group map, so it must outlive both maps below. + std::unordered_map _current_offset_indexes; std::map> _current_predicate_columns; // predicate ColumnReaders std::map> @@ -204,7 +206,6 @@ class ParquetScanScheduler { int64_t _current_row_group_first_row = 0; // first file row of the current row group std::vector _current_selected_ranges; // selected ranges for the current row group after page-index pruning - std::unordered_map _current_offset_indexes; size_t _current_range_idx = 0; // current selected_range index int64_t _current_range_rows_read = 0; // rows read in the current range // Predicate readers move immediately because they decide which rows survive. Non-predicate diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp index 1706039f03bc7f..761a7f6a591f8a 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -100,7 +100,8 @@ CountColumnReader::~CountColumnReader() { Status CountColumnReader::create(io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, const ParquetColumnSchema& root_schema, const format::LocalColumnIndex* projection, io::IOContext* io_ctx, - bool enable_page_cache, ParquetColumnReaderProfile profile, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, std::unique_ptr* reader) { DORIS_CHECK(file != nullptr); DORIS_CHECK(metadata != nullptr); @@ -136,7 +137,7 @@ Status CountColumnReader::create(io::FileReaderSPtr file, const FileMetaData* me RETURN_IF_ERROR(native::LevelReader::create( std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, - enable_page_cache, &level_reader)); + enable_page_cache, page_cache_file_key, &level_reader)); reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h index 86508e4302d1ea..cd8ecb09473a53 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.h +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -44,7 +44,8 @@ class CountColumnReader { static Status create(io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, const ParquetColumnSchema& root_schema, const format::LocalColumnIndex* projection, io::IOContext* io_ctx, - bool enable_page_cache, ParquetColumnReaderProfile profile, + bool enable_page_cache, const std::string& page_cache_file_key, + ParquetColumnReaderProfile profile, std::unique_ptr* reader); Status skip(int64_t rows); diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp index d5c9d2f2ff6d4a..97d7608e1ba477 100644 --- a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -79,7 +79,10 @@ Status BoolPlainDecoder::skip_values(size_t num_values) { int num_remaining = cast_set(num_values - skip_cached); int num_to_skip = BitUtil::RoundDownToPowerOf2(num_remaining, 32); if (num_to_skip > 0) { - bool_values_.SkipBatch(1, num_to_skip); + // A failed bulk skip must not be reported as success for a truncated boolean page. + if (!bool_values_.SkipBatch(1, num_to_skip)) { + return Status::IOError("Can't skip enough booleans in plain decoder"); + } } num_remaining -= num_to_skip; if (num_remaining > 0) { diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h index 7a6806d4acd91f..35d076baccf3e9 100644 --- a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -59,6 +59,13 @@ class BoolPlainDecoder final : public Decoder { Status skip_values(size_t num_values) override; + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&selected_values_, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return selected_values_.capacity() * sizeof(uint8_t); + } + protected: inline bool _decode_value(bool* value) { if (LIKELY(unpacked_value_idx_ < num_unpacked_values_)) { diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp index d1923c83c9f8bc..8de0ab7b9628ab 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -48,18 +48,22 @@ Status BoolRLEDecoder::set_data(Slice* slice) { } _num_bytes = num_bytes; auto decoder_data = data + 4; - _decoder = RleDecoder(decoder_data, num_bytes, 1); + _decoder = RleBatchDecoder(const_cast(decoder_data), num_bytes, 1); return Status::OK(); } Status BoolRLEDecoder::skip_values(size_t num_values) { - _decoder.Skip(num_values); + _values.resize(num_values); + // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. + if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { + return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + } return Status::OK(); } Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { _values.resize(num_values); - if (!_decoder.get_values(_values.data(), num_values)) { + if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { return Status::IOError("Can't read enough booleans in Parquet RLE decoder"); } return consumer.consume(_values.data(), _values.size(), sizeof(uint8_t)); @@ -68,7 +72,8 @@ Status BoolRLEDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueC Status BoolRLEDecoder::decode_selected_fixed_values(const ParquetSelection& selection, ParquetFixedValueConsumer& consumer) { _values.resize(selection.total_values); - if (!_decoder.get_values(_values.data(), selection.total_values)) { + if (_decoder.GetBatch(_values.data(), cast_set(selection.total_values)) != + selection.total_values) { return Status::IOError("Can't read enough booleans in Parquet RLE selection decoder"); } size_t output = 0; diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h index e05a33d5e77e8b..11e2acd35383af 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -46,8 +46,13 @@ class BoolRLEDecoder final : public Decoder { Status skip_values(size_t num_values) override; + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(uint8_t); } + private: - RleDecoder _decoder; + RleBatchDecoder _decoder; std::vector _values; size_t _num_bytes; }; diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h index 77ba88875ed5c5..0ad27cb2dd6a13 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -52,6 +52,13 @@ class ByteArrayPlainDecoder final : public Decoder { Status skip_values(size_t num_values) override; + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_binary_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _binary_values.capacity() * sizeof(StringRef); + } + private: std::vector _binary_values; }; diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp index 783834cbfa7c2e..caf3b675e2dfd7 100644 --- a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.cpp @@ -30,7 +30,6 @@ Status ByteStreamSplitDecoder::decode_fixed_values(size_t num_values, if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { return Status::IOError("Out-of-bounds access in Parquet byte-stream-split decoder"); } - DORIS_CHECK_EQ(_data->size % static_cast(_type_length), 0); const int64_t stride = static_cast(_data->size / _type_length); _decoded_values.resize(byte_size); byte_stream_split_decode(reinterpret_cast(_data->data), _type_length, @@ -69,12 +68,14 @@ Status ByteStreamSplitDecoder::decode_selected_fixed_values(const ParquetSelecti } Status ByteStreamSplitDecoder::skip_values(size_t num_values) { - _offset += _type_length * num_values; - if (UNLIKELY(_offset > _data->size)) { + // Check in row units before multiplication so a corrupt skip count cannot wrap back in-bounds. + if (UNLIKELY(_type_length <= 0 || _offset > _data->size || + num_values > (_data->size - _offset) / _type_length)) { return Status::IOError( "Out-of-bounds access in parquet data decoder: offset = {}, size = {}", _offset, _data->size); } + _offset += _type_length * num_values; return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h index d6d190742939d6..89cf1140d46cb0 100644 --- a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -27,6 +27,15 @@ class ByteStreamSplitDecoder final : public Decoder { ByteStreamSplitDecoder() = default; ~ByteStreamSplitDecoder() override = default; + Status set_data(Slice* data) override { + if (UNLIKELY(_type_length <= 0 || data == nullptr || + data->size % static_cast(_type_length) != 0)) { + // Every byte lane must contain the same number of complete physical values. + return Status::Corruption("Invalid Parquet byte-stream-split page length"); + } + return Decoder::set_data(data); + } + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override; Status decode_selected_fixed_values(const ParquetSelection& selection, @@ -34,6 +43,13 @@ class ByteStreamSplitDecoder final : public Decoder { Status skip_values(size_t num_values) override; + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_decoded_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _decoded_values.capacity() * sizeof(uint8_t); + } + private: std::vector _decoded_values; }; diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index fae204b4c6e277..a6f28248c481b5 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -328,14 +328,19 @@ Status ColumnChunkReader::next_page() { } template -void ColumnChunkReader::_get_uncompressed_levels( +Status ColumnChunkReader::_get_uncompressed_levels( const tparquet::DataPageHeaderV2& page_v2, Slice& page_data) { - int32_t rl = page_v2.repetition_levels_byte_length; - int32_t dl = page_v2.definition_levels_byte_length; + const size_t rl = page_v2.repetition_levels_byte_length; + const size_t dl = page_v2.definition_levels_byte_length; + if (UNLIKELY(rl > page_data.size || dl > page_data.size - rl)) { + // Validate the physical slice again because a cached entry may itself be truncated. + return Status::Corruption("Parquet data page v2 level bytes exceed available payload"); + } _v2_rep_levels = Slice(page_data.data, rl); _v2_def_levels = Slice(page_data.data + rl, dl); page_data.data += dl + rl; page_data.size -= dl + rl; + return Status::OK(); } template @@ -366,16 +371,31 @@ Status ColumnChunkReader::load_page_data() { size_t rl = header_v2.repetition_levels_byte_length; size_t dl = header_v2.definition_levels_byte_length; levels_size = rl + dl; + if (UNLIKELY(header_size > cached.size || + levels_size > cached.size - header_size)) { + return Status::Corruption("Cached Parquet page is shorter than its v2 levels"); + } _v2_rep_levels = Slice(reinterpret_cast(cached.data) + header_size, rl); _v2_def_levels = Slice(reinterpret_cast(cached.data) + header_size + rl, dl); } // payload_slice points to the bytes after header and levels + if (UNLIKELY(header_size + levels_size > cached.size)) { + return Status::Corruption("Cached Parquet page is shorter than its header"); + } Slice payload_slice(cached.data + header_size + levels_size, cached.size - header_size - levels_size); bool cache_payload_is_decompressed = _page_reader->is_cache_payload_decompressed(); + const size_t expected_payload_size = + cache_payload_is_decompressed + ? static_cast(header->uncompressed_page_size) - levels_size + : static_cast(header->compressed_page_size) - levels_size; + if (UNLIKELY(payload_slice.size != expected_payload_size)) { + return Status::Corruption("Cached Parquet page payload has size {}, expected {}", + payload_slice.size, expected_payload_size); + } if (cache_payload_is_decompressed) { // Cached payload is already uncompressed @@ -419,7 +439,7 @@ Status ColumnChunkReader::load_page_data() { memcpy(level_bytes.data(), compressed_data.data, level_sz); } // now remove levels from compressed_data for decompression - _get_uncompressed_levels(header_v2, compressed_data); + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, compressed_data)); } bool is_v2_compressed = header->__isset.data_page_header_v2 && header->data_page_header_v2.is_compressed; @@ -476,7 +496,7 @@ Status ColumnChunkReader::load_page_data() { level_bytes.resize(level_sz); memcpy(level_bytes.data(), uncompressed_data.data, level_sz); } - _get_uncompressed_levels(header_v2, uncompressed_data); + RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, uncompressed_data)); } // copy page data out _page_data = Slice(uncompressed_data.data, uncompressed_data.size); @@ -531,6 +551,9 @@ Status ColumnChunkReader::load_page_data() { _decoders[static_cast(encoding)] = std::move(page_decoder); _page_decoder = _decoders[static_cast(encoding)].get(); } + // Encoding headers cannot legitimately advertise more physical values than the data page's + // logical value count; establish the bound before decoders inspect external counts. + _page_decoder->set_expected_values(_remaining_num_values); RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); _state = DATA_LOADED; @@ -573,6 +596,11 @@ Status ColumnChunkReader::_decode_dict_page() { if (cache_payload_is_decompressed) { // Use cached decompressed dictionary data + if (UNLIKELY(payload_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Cached Parquet dictionary payload has size {}, expected {}", + payload_slice.size, uncompressed_size); + } memcpy(dict_data.get(), payload_slice.data, payload_slice.size); dict_loaded = true; } else { @@ -709,13 +737,13 @@ Status ColumnChunkReader::skip_values(size_t num_va return Status::IOError("Skip too many values in current page. {} vs. {}", _remaining_num_values, num_values); } - _remaining_num_values -= num_values; if (skip_data) { SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); - return _page_decoder->skip_values(num_values); - } else { - return Status::OK(); + RETURN_IF_ERROR(_page_decoder->skip_values(num_values)); } + // Commit logical page progress only after the physical decoder accepted the whole request. + _remaining_num_values -= num_values; + return Status::OK(); } template @@ -733,23 +761,29 @@ Status ColumnChunkReader::materialize_values( if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { return Status::IOError("Decode too many values in current page"); } - _remaining_num_values -= select_vector.num_values(); RETURN_IF_ERROR(translate_value_encoding(_current_encoding, &context.encoding)); + Status status; if (select_vector.has_filter()) { if (select_vector.num_nulls() == 0) { ++_chunk_statistics.hybrid_selection_batches; - const Status status = decode_selected_non_null_values( - *doris_column, serde, *_page_decoder, context, state, select_vector, - &_chunk_statistics.materialization_time); + status = decode_selected_non_null_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); - return status; + } else { + ++_chunk_statistics.hybrid_selection_null_fallback_batches; + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, + state, select_vector, + &_chunk_statistics.materialization_time); } - ++_chunk_statistics.hybrid_selection_null_fallback_batches; - return decode_selected_values(*doris_column, serde, *_page_decoder, context, state, - select_vector, &_chunk_statistics.materialization_time); + } else { + status = decode_selected_values(*doris_column, serde, *_page_decoder, context, state, + select_vector, + &_chunk_statistics.materialization_time); } - return decode_selected_values(*doris_column, serde, *_page_decoder, context, state, - select_vector, &_chunk_statistics.materialization_time); + RETURN_IF_ERROR(status); + _remaining_num_values -= select_vector.num_values(); + return Status::OK(); } template @@ -861,6 +895,9 @@ Status ColumnChunkReader::load_page_nested_rows( rep_levels.reserve(rep_levels.size() + _remaining_rep_nums); while (_remaining_rep_nums) { level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } if (rep_level == 0) { // rep_level 0 indicates start of new row if (*result_rows == max_rows) { // this page contain max_rows, page no end. _current_row += max_rows; @@ -892,6 +929,9 @@ Status ColumnChunkReader::load_cross_page_nested_ro *cross_page = has_next_page(); while (_remaining_rep_nums) { level_t rep_level = _rep_level_get_next(); + if (UNLIKELY(rep_level < 0)) { + return Status::Corruption("Parquet repetition level stream ended unexpectedly"); + } if (rep_level == 0) { // rep_level 0 indicates start of new row *cross_page = false; _rep_level_rewind_one(); diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 3cc5dc35910380..f48755050c406f 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -66,6 +66,7 @@ struct ColumnChunkReaderStatistics { int64_t parse_page_header_num = 0; int64_t read_page_header_time = 0; int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; int64_t page_cache_write_counter = 0; int64_t page_cache_compressed_write_counter = 0; int64_t page_cache_decompressed_write_counter = 0; @@ -153,6 +154,15 @@ class ColumnChunkReader { // Get page decoder Decoder* get_page_decoder() { return _page_decoder; } + void release_decoder_scratch(size_t max_retained_bytes) { + for (auto& [encoding, decoder] : _decoders) { + decoder->release_scratch(max_retained_bytes); + } + // Level decoders may batch-convert unsigned RLE values into Doris' signed level_t. + _rep_level_decoder.release_scratch(max_retained_bytes); + _def_level_decoder.release_scratch(max_retained_bytes); + } + tparquet::Encoding::type current_encoding() const { return _current_encoding; } ColumnChunkReaderStatistics& chunk_statistics() { @@ -168,6 +178,8 @@ class ColumnChunkReader { // ColumnChunkReader because insertion happens after decompression and therefore remain in // the chunk accumulator above. _chunk_statistics.page_read_counter = _page_reader->page_statistics().page_read_counter; + _chunk_statistics.data_page_read_counter = + _page_reader->page_statistics().data_page_read_counter; _chunk_statistics.page_cache_hit_counter = _page_reader->page_statistics().page_cache_hit_counter; _chunk_statistics.page_cache_missing_counter = @@ -203,7 +215,13 @@ class ColumnChunkReader { def_values.resize(before_sz + append_sz, 0); if (max_def_level() != 0) { auto ptr = def_values.data() + before_sz; - _def_level_decoder.get_levels(ptr, append_sz); + const size_t decoded = _def_level_decoder.get_levels(ptr, append_sz); + if (UNLIKELY(decoded != append_sz)) { + def_values.resize(before_sz); + return Status::Corruption( + "Parquet definition level stream ended after {} of {} slots", decoded, + append_sz); + } } _remaining_def_nums -= append_sz; return Status::OK(); @@ -229,7 +247,7 @@ class ColumnChunkReader { int32_t _get_type_length(); void _insert_page_into_cache(const std::vector& level_bytes, const Slice& payload); - void _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); + Status _get_uncompressed_levels(const tparquet::DataPageHeaderV2& page_v2, Slice& page_data); Status _skip_nested_rows_in_page(size_t num_rows); level_t _rep_level_get_next() { diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 77ca37bbaecdb7..8b92749c0c80af 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -97,6 +97,10 @@ size_t retained_set_bytes(const std::unordered_set& values) { return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); } +bool is_direct_binary_type(PrimitiveType type) { + return is_string_type(type) || type == TYPE_VARBINARY; +} + // The target SerDe can fuse physical decode with these logical type changes. Less common schema // changes retain the generic file-format converter as a compatibility path: the decoder still // exposes raw spans, but the source SerDe first materializes a reusable source column before the @@ -108,7 +112,10 @@ bool serde_can_materialize_directly(const DataTypePtr& source_type, return source == target || (is_direct_integer_type(source) && is_direct_integer_type(target)) || (source == TYPE_FLOAT && target == TYPE_DOUBLE) || (is_direct_decimal_type(source) && is_direct_decimal_type(target)) || - (is_string_type(source) && is_string_type(target)); + // Parquet STRING and VARBINARY share BYTE_ARRAY bytes. Materializing through the target + // SerDe preserves those bytes and avoids a converter whose scratch column uses the v1 + // String representation instead of the native ColumnVarbinary representation. + (is_direct_binary_type(source) && is_direct_binary_type(target)); } Status init_decode_context(const FieldSchema& field, const cctz::time_zone* ctz, @@ -279,17 +286,20 @@ static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, null_map.resize(pos); } -static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offsets_data, - NullMap* null_map_ptr, const std::vector& rep_levels, - const std::vector& def_levels) { +static Status fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offsets_data, + NullMap* null_map_ptr, const std::vector& rep_levels, + const std::vector& def_levels) { size_t num_levels = rep_levels.size(); - DCHECK_EQ(num_levels, def_levels.size()); + if (UNLIKELY(num_levels != def_levels.size())) { + return Status::Corruption("Parquet repetition and definition level counts differ"); + } size_t origin_size = offsets_data.size(); offsets_data.resize(origin_size + num_levels); if (null_map_ptr != nullptr) { null_map_ptr->resize(origin_size + num_levels); } size_t offset_pos = origin_size - 1; + bool parent_opened = false; for (size_t i = 0; i < num_levels; ++i) { // skip the levels affect its ancestor or its descendants if (def_levels[i] < field->repeated_parent_def_level || @@ -297,9 +307,15 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset continue; } if (rep_levels[i] == field->repetition_level) { + // A continuation can extend only a parent opened by this aligned logical batch. + if (UNLIKELY(!parent_opened)) { + return Status::Corruption( + "Parquet collection starts with an orphan repetition continuation"); + } offsets_data[offset_pos]++; continue; } + parent_opened = true; offset_pos++; offsets_data[offset_pos] = offsets_data[offset_pos - 1]; if (def_levels[i] >= field->definition_level) { @@ -317,22 +333,24 @@ static void fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offset if (null_map_ptr != nullptr) { null_map_ptr->resize(offset_pos + 1); } + return Status::OK(); } Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, const tparquet::RowGroup& row_group, const RowRanges& row_ranges, const cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, size_t max_buf_size, - std::unordered_map& col_offsets, + const std::unordered_map& col_offsets, RuntimeState* state, bool in_collection, const std::set& column_ids, - const std::set& filter_column_ids) { + const std::set& filter_column_ids, + const std::string& page_cache_file_key) { size_t total_rows = row_group.num_rows; if (field->data_type->get_primitive_type() == TYPE_ARRAY) { std::unique_ptr element_reader; RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, element_reader, max_buf_size, col_offsets, state, true, column_ids, - filter_column_ids)); + filter_column_ids, page_cache_file_key)); auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); element_reader->set_column_in_nested(); RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); @@ -347,7 +365,7 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, // Create key reader RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, key_reader, max_buf_size, col_offsets, state, true, column_ids, - filter_column_ids)); + filter_column_ids, page_cache_file_key)); } else { auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, io_ctx, &field->children[0]); @@ -359,7 +377,7 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, // Create value reader RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, value_reader, max_buf_size, col_offsets, state, true, column_ids, - filter_column_ids)); + filter_column_ids, page_cache_file_key)); } else { auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, io_ctx, &field->children[1]); @@ -382,7 +400,8 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, if (column_ids.empty() || column_ids.find(child.get_column_id()) != column_ids.end()) { RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, child_reader, max_buf_size, col_offsets, state, - in_collection, column_ids, filter_column_ids)); + in_collection, column_ids, filter_column_ids, + page_cache_file_key)); child_readers[child.name] = std::move(child_reader); // Record the first non-SkippingReader if (non_skip_reader_idx == -1) { @@ -401,7 +420,7 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, std::unique_ptr child_reader; RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, child_reader, max_buf_size, col_offsets, state, in_collection, - column_ids, filter_column_ids)); + column_ids, filter_column_ids, page_cache_file_key)); child_reader->set_column_in_nested(); child_readers[field->children[0].name] = std::move(child_reader); } @@ -411,9 +430,9 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, reader.reset(struct_reader.release()); } else { auto physical_index = field->physical_column_index; + const auto offset_it = col_offsets.find(physical_index); const tparquet::OffsetIndex* offset_index = - col_offsets.find(physical_index) != col_offsets.end() ? &col_offsets[physical_index] - : nullptr; + offset_it != col_offsets.end() ? &offset_it->second : nullptr; const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; if (in_collection) { @@ -421,14 +440,16 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + RETURN_IF_ERROR( + scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } else { auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + RETURN_IF_ERROR( + scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } @@ -437,14 +458,16 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + RETURN_IF_ERROR( + scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } else { auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state)); + RETURN_IF_ERROR( + scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } @@ -459,10 +482,9 @@ void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* res } template -Status ScalarColumnReader::init(io::FileReaderSPtr file, - FieldSchema* field, - size_t max_buf_size, - RuntimeState* state) { +Status ScalarColumnReader::init( + io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, RuntimeState* state, + const std::string& page_cache_file_key) { _field_schema = field; auto& chunk_meta = _chunk_meta.meta_data; int64_t chunk_start = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset @@ -479,7 +501,8 @@ Status ScalarColumnReader::init(io::FileReaderSPtr _stream_reader = std::make_unique(file, chunk_start, chunk_len, prefetch_buffer_size); ParquetPageReadContext ctx( - (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache); + (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache, + page_cache_file_key); _chunk_reader = std::make_unique>( _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx); @@ -491,6 +514,10 @@ Status ScalarColumnReader::init(io::FileReaderSPtr template void ScalarColumnReader::release_batch_scratch( size_t max_retained_bytes) { + if (_chunk_reader != nullptr) { + // Persistent decoders also own batch-sized value/slice buffers, not only the reader. + _chunk_reader->release_decoder_scratch(max_retained_bytes); + } bool release_selection = false; release_selection |= release_vector_if_oversized(&_rep_levels, max_retained_bytes); release_selection |= release_vector_if_oversized(&_def_levels, max_retained_bytes); @@ -554,15 +581,7 @@ Status ScalarColumnReader::_skip_values(size_t num_ level_t def_level = -1; size_t loop_skip = def_decoder.get_next_run(&def_level, num_values - skipped); if (loop_skip == 0) { - std::stringstream ss; - auto& bit_reader = def_decoder.rle_decoder().bit_reader(); - ss << "def_decoder buffer (hex): "; - for (size_t i = 0; i < bit_reader.max_bytes(); ++i) { - ss << std::hex << std::setw(2) << std::setfill('0') - << static_cast(bit_reader.buffer()[i]) << " "; - } - LOG(WARNING) << ss.str(); - return Status::InternalError("Failed to decode definition level."); + return Status::Corruption("Parquet definition level stream ended while skipping"); } if (def_level < _field_schema->definition_level) { null_size += loop_skip; @@ -611,15 +630,8 @@ Status ScalarColumnReader::_read_values(size_t num_ level_t def_level; size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); if (loop_read == 0) { - std::stringstream ss; - auto& bit_reader = def_decoder.rle_decoder().bit_reader(); - ss << "def_decoder buffer (hex): "; - for (size_t i = 0; i < bit_reader.max_bytes(); ++i) { - ss << std::hex << std::setw(2) << std::setfill('0') - << static_cast(bit_reader.buffer()[i]) << " "; - } - LOG(WARNING) << ss.str(); - return Status::InternalError("Failed to decode definition level."); + return Status::Corruption( + "Parquet definition level stream ended while materializing"); } bool is_null = def_level < _field_schema->definition_level; @@ -1090,9 +1102,12 @@ Status ArrayColumnReader::read_column_data( ColumnArray::Offsets64& offsets_data = assert_cast(*data_column).get_offsets(); // fill offset and null map - fill_array_offset(_field_schema, offsets_data, null_map_ptr, _element_reader->get_rep_level(), - _element_reader->get_def_level()); - DCHECK_EQ(element_column->size(), offsets_data.back()); + RETURN_IF_ERROR(fill_array_offset(_field_schema, offsets_data, null_map_ptr, + _element_reader->get_rep_level(), + _element_reader->get_def_level())); + if (UNLIKELY(element_column->size() != offsets_data.back())) { + return Status::Corruption("Parquet array element count does not match repetition levels"); + } #ifndef NDEBUG doris_column->sanity_check(); #endif @@ -1158,7 +1173,11 @@ Status MapColumnReader::read_column_data( key_column->size() - orig_col_column_size)); value_rows += loop_rows; } - DCHECK_EQ(key_rows, value_rows); + if (UNLIKELY(key_rows != value_rows)) { + // MAP children share one logical-row boundary; EOF in one sibling is file corruption. + return Status::Corruption("Parquet map value reader returned {} rows for {} key rows", + value_rows, key_rows); + } *read_rows = key_rows; *eof = key_eof; @@ -1166,11 +1185,28 @@ Status MapColumnReader::read_column_data( return Status::OK(); } - DCHECK_EQ(key_column->size(), value_column->size()); + const size_t key_values = key_column->size() - orig_col_column_size; + if (UNLIKELY(key_column->size() != value_column->size())) { + return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", + key_column->size(), value_column->size()); + } + if (const auto* nullable_keys = typeid_cast(key_column.get()); UNLIKELY( + nullable_keys != nullptr && + nullable_keys->has_null(orig_col_column_size, orig_col_column_size + key_values))) { + // Doris MAP keys are non-null even if a malformed/evolved file exposes a nullable child. + return Status::Corruption("Parquet map contains a null key"); + } + const auto& key_rep_levels = _key_reader->get_rep_level(); + const auto& value_rep_levels = _value_reader->get_rep_level(); + if (UNLIKELY(key_rep_levels != value_rep_levels)) { + return Status::Corruption("Parquet map key/value repetition shapes differ"); + } // fill offset and null map - fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, _key_reader->get_rep_level(), - _key_reader->get_def_level()); - DCHECK_EQ(key_column->size(), map.get_offsets().back()); + RETURN_IF_ERROR(fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, + key_rep_levels, _key_reader->get_def_level())); + if (UNLIKELY(key_column->size() != map.get_offsets().back())) { + return Status::Corruption("Parquet map entry count does not match repetition levels"); + } #ifndef NDEBUG doris_column->sanity_check(); #endif @@ -1296,7 +1332,11 @@ Status StructColumnReader::read_column_data( is_dict_filter)); field_rows += loop_rows; } - DCHECK_EQ(*read_rows, field_rows); + if (UNLIKELY(*read_rows != field_rows)) { + // STRUCT siblings must advance the same logical rows before any result is exposed. + return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", + file_name, field_rows, *read_rows); + } // DCHECK_EQ(*eof, field_eof); } } diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index e7ca4093707c2f..1a1c195bab4e83 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -103,7 +103,8 @@ class ColumnReader { page_cache_hit_counter(cs.page_cache_hit_counter), page_cache_missing_counter(cs.page_cache_missing_counter), page_cache_compressed_hit_counter(cs.page_cache_compressed_hit_counter), - page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter) {} + page_cache_decompressed_hit_counter(cs.page_cache_decompressed_hit_counter), + leaf_page_read_counters {cs.data_page_read_counter} {} int64_t page_index_read_calls; int64_t decompress_time; @@ -129,6 +130,8 @@ class ColumnReader { int64_t page_cache_missing_counter; int64_t page_cache_compressed_hit_counter; int64_t page_cache_decompressed_hit_counter; + // Preserve per-leaf identity when complex readers aggregate their other counters. + std::vector leaf_page_read_counters; void merge(ColumnStatistics& col_statistics) { page_index_read_calls += col_statistics.page_index_read_calls; @@ -149,6 +152,9 @@ class ColumnReader { parse_page_header_num += col_statistics.parse_page_header_num; read_page_header_time += col_statistics.read_page_header_time; page_read_counter += col_statistics.page_read_counter; + leaf_page_read_counters.insert(leaf_page_read_counters.end(), + col_statistics.leaf_page_read_counters.begin(), + col_statistics.leaf_page_read_counters.end()); page_cache_write_counter += col_statistics.page_cache_write_counter; page_cache_compressed_write_counter += col_statistics.page_cache_compressed_write_counter; @@ -188,10 +194,11 @@ class ColumnReader { const tparquet::RowGroup& row_group, const RowRanges& row_ranges, const cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, size_t max_buf_size, - std::unordered_map& col_offsets, + const std::unordered_map& col_offsets, RuntimeState* state, bool in_collection = false, const std::set& column_ids = {}, - const std::set& filter_column_ids = {}); + const std::set& filter_column_ids = {}, + const std::string& page_cache_file_key = {}); virtual const std::vector& get_rep_level() const = 0; virtual const std::vector& get_def_level() const = 0; virtual ColumnStatistics column_statistics() = 0; @@ -239,7 +246,7 @@ class ScalarColumnReader : public ColumnReader { _offset_index(offset_index) {} ~ScalarColumnReader() override { close(); } Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, - RuntimeState* state); + RuntimeState* state, const std::string& page_cache_file_key); Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index d1e861f20d0211..3b031a68a98190 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,9 @@ class Decoder : public ParquetDecodeSource { return Status::OK(); } + // Page headers provide an upper bound before encoding-specific headers are trusted. + virtual void set_expected_values(size_t expected_values) { _expected_values = expected_values; } + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { return Status::NotSupported("Fixed values are not supported by this Parquet decoder"); } @@ -63,15 +67,26 @@ class Decoder : public ParquetDecodeSource { Status skip_values(size_t num_values) override = 0; + virtual void release_scratch(size_t max_retained_bytes) {} + virtual size_t retained_scratch_bytes() const { return 0; } + virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, size_t num_values) { return Status::NotSupported("set_dict is not supported"); } protected: - int32_t _type_length; + template + static void release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { + if (values->capacity() * sizeof(T) > max_retained_bytes) { + std::vector().swap(*values); + } + } + + int32_t _type_length = -1; Slice* _data = nullptr; uint32_t _offset = 0; + size_t _expected_values = std::numeric_limits::max(); }; class BaseDictDecoder : public Decoder { @@ -87,6 +102,12 @@ class BaseDictDecoder : public Decoder { _data = data; _offset = 0; uint8_t bit_width = *data->data; + // Dictionary indices are uint32_t; wider external widths make repeated runs overwrite the + // decoder's four-byte state before any dictionary-bound check can run. + if (UNLIKELY(bit_width > 32)) { + return Status::Corruption("Parquet dictionary index bit width {} exceeds 32", + bit_width); + } _index_batch_decoder = std::make_unique>( reinterpret_cast(data->data) + 1, static_cast(data->size) - 1, bit_width); @@ -143,6 +164,13 @@ class BaseDictDecoder : public Decoder { return Status::OK(); } + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_skip_indices, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _skip_indices.capacity() * sizeof(uint32_t); + } + protected: Status skip_values(size_t num_values) override { _skip_indices.resize(num_values); diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp index dac29ccfc5d4d4..78a12593667cfd 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -21,13 +21,16 @@ namespace doris::format::parquet::native { Status DeltaLengthByteArrayDecoder::_decode_lengths() { RETURN_IF_ERROR(_len_decoder.set_bit_reader(_bit_reader)); // get the number of encoded lengths - int num_length = _len_decoder.valid_values_count(); + uint32_t num_length = _len_decoder.valid_values_count(); _buffered_length.resize(num_length); // decode all the lengths. all the lengths are buffered in buffered_length_. uint32_t ret; RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), num_length, &ret)); - DCHECK_EQ(ret, num_length); + if (UNLIKELY(ret != num_length)) { + return Status::Corruption("Parquet delta length stream decoded {} of {} lengths", ret, + num_length); + } _length_idx = 0; _num_valid_values = num_length; return Status::OK(); @@ -43,7 +46,7 @@ Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, return Status::OK(); } - int32_t data_size = 0; + int64_t data_size = 0; const int32_t* length_ptr = _buffered_length.data() + _length_idx; for (int i = 0; i < max_values; ++i) { int32_t len = length_ptr[i]; @@ -51,15 +54,21 @@ Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, return Status::InvalidArgument("Negative string delta length"); } buffer[i].size = len; - if (common::add_overflow(data_size, len, data_size)) { + if (common::add_overflow(data_size, static_cast(len), data_size)) { return Status::InvalidArgument("Excess expansion in DELTA_(LENGTH_)BYTE_ARRAY"); } } + // Every declared byte must exist in this page. Check before resize so a tiny malformed stream + // cannot reserve memory based only on attacker-controlled decoded lengths. + if (UNLIKELY(data_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + data_size, _bit_reader->bytes_left()); + } _length_idx += max_values; _buffered_data.resize(data_size); char* data_ptr = _buffered_data.data(); - for (int j = 0; j < data_size; j++) { + for (int64_t j = 0; j < data_size; j++) { if (!_bit_reader->GetValue(8, data_ptr + j)) { return Status::IOError("Get length bytes EOF"); } @@ -130,4 +139,4 @@ Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* *out_num_values = max_values; return Status::OK(); } -} // namespace doris::format::parquet::native \ No newline at end of file +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index 13d041b033f025..68c8ebd7e5a44c 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -67,8 +67,15 @@ class DeltaBitPackDecoder final : public DeltaDecoder { Status skip_values(size_t num_values) override { _values.resize(num_values); - uint32_t num_valid_values; - return _get_internal(_values.data(), cast_set(num_values), &num_valid_values); + uint32_t num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &num_valid_values)); + // Skips have the same exact-consumption contract as materialized decodes. + if (UNLIKELY(num_valid_values != num_values)) { + return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", + num_values, num_valid_values); + } + return Status::OK(); } Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { @@ -113,6 +120,14 @@ class DeltaBitPackDecoder final : public DeltaDecoder { return _total_values_remaining; } + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); + } + Status set_data(Slice* slice) override { _bit_reader.reset( new BitReader((const uint8_t*)slice->data, cast_set(slice->size))); @@ -166,10 +181,21 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { explicit DeltaLengthByteArrayDecoder() : _len_decoder(), _buffered_length(0), _buffered_data(0) {} + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _len_decoder.set_expected_values(expected_values); + } + Status skip_values(size_t num_values) override { _values.resize(num_values); - int num_valid_values; - return _get_internal(_values.data(), cast_set(num_values), &num_valid_values); + int num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), cast_set(num_values), &num_valid_values)); + if (UNLIKELY(num_valid_values != num_values)) { + return Status::IOError("Expected to skip {} Parquet delta-length values, skipped {}", + num_values, num_valid_values); + } + return Status::OK(); } Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { @@ -214,9 +240,30 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { return _get_internal(buffer, num_values, out_num_values); } + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_buffered_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _len_decoder.release_scratch(max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _buffered_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _len_decoder.retained_scratch_bytes(); + } + Status set_data(Slice* slice) override { - if (slice->size == 0) { - return Status::OK(); + if (slice == nullptr || slice->size == 0) { + // Reused decoders must never retain lengths or payload pointers from the prior page. + _bit_reader.reset(); + _data = nullptr; + _offset = 0; + _num_valid_values = 0; + _length_idx = 0; + _buffered_length.clear(); + _buffered_data.clear(); + return Status::Corruption("Parquet delta-length page is empty"); } _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); _data = slice; @@ -252,10 +299,15 @@ class DeltaByteArrayDecoder : public DeltaDecoder { public: explicit DeltaByteArrayDecoder() : _buffered_prefix_length(0), _buffered_data(0) {} + void set_expected_values(size_t expected_values) override { + Decoder::set_expected_values(expected_values); + _prefix_len_decoder.set_expected_values(expected_values); + _suffix_decoder.set_expected_values(expected_values); + } + Status skip_values(size_t num_values) override { - _values.resize(num_values); - int num_valid_values; - return _get_internal(_values.data(), cast_set(num_values), &num_valid_values); + RETURN_IF_ERROR(_decode_slices(num_values)); + return _validate_fixed_width_values(); } Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { @@ -284,13 +336,10 @@ class DeltaByteArrayDecoder : public DeltaDecoder { Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { RETURN_IF_ERROR(_decode_slices(num_values)); + RETURN_IF_ERROR(_validate_fixed_width_values()); const size_t byte_size = num_values * static_cast(_type_length); _fixed_values.resize(byte_size); for (size_t row = 0; row < num_values; ++row) { - if (UNLIKELY(_values[row].size != static_cast(_type_length))) { - return Status::Corruption("Parquet fixed value has length {}, expected {}", - _values[row].size, _type_length); - } memcpy(_fixed_values.data() + row * _type_length, _values[row].data, _type_length); } return consumer.consume(_fixed_values.data(), num_values, @@ -300,6 +349,9 @@ class DeltaByteArrayDecoder : public DeltaDecoder { Status decode_selected_fixed_values(const ParquetSelection& selection, ParquetFixedValueConsumer& consumer) override { RETURN_IF_ERROR(_decode_slices(selection.total_values)); + // Validate every consumed value, including filtered ranges, so selection cannot hide a + // malformed FIXED_LEN_BYTE_ARRAY width that a later cursor advance has already committed. + RETURN_IF_ERROR(_validate_fixed_width_values()); DORIS_CHECK(_type_length > 0); const size_t value_width = static_cast(_type_length); if (UNLIKELY(selection.selected_values > @@ -311,10 +363,6 @@ class DeltaByteArrayDecoder : public DeltaDecoder { for (const auto& range : selection.ranges) { for (size_t row = 0; row < range.count; ++row) { const auto& value = _values[range.first + row]; - if (UNLIKELY(value.size != value_width)) { - return Status::Corruption("Parquet fixed value has length {}, expected {}", - value.size, value_width); - } memcpy(_fixed_values.data() + output * value_width, value.data, value_width); ++output; } @@ -352,7 +400,44 @@ class DeltaByteArrayDecoder : public DeltaDecoder { return _get_internal(buffer, num_values, out_num_values); } + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_values, max_retained_bytes); + release_vector_if_oversized(&_string_refs, max_retained_bytes); + release_vector_if_oversized(&_fixed_values, max_retained_bytes); + release_vector_if_oversized(&_buffered_prefix_length, max_retained_bytes); + release_vector_if_oversized(&_buffered_data, max_retained_bytes); + _prefix_len_decoder.release_scratch(max_retained_bytes); + _suffix_decoder.release_scratch(max_retained_bytes); + if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); + if (_last_value_in_previous_page.capacity() > max_retained_bytes) { + std::string().swap(_last_value_in_previous_page); + } + } + size_t retained_scratch_bytes() const override { + return _values.capacity() * sizeof(Slice) + _string_refs.capacity() * sizeof(StringRef) + + _fixed_values.capacity() * sizeof(uint8_t) + + _buffered_prefix_length.capacity() * sizeof(int32_t) + + _buffered_data.capacity() * sizeof(char) + _last_value.capacity() + + _last_value_in_previous_page.capacity() + + _prefix_len_decoder.retained_scratch_bytes() + + _suffix_decoder.retained_scratch_bytes(); + } + private: + Status _validate_fixed_width_values() const { + if (_type_length <= 0) { + return Status::OK(); + } + const size_t value_width = static_cast(_type_length); + for (const auto& value : _values) { + if (UNLIKELY(value.size != value_width)) { + return Status::Corruption("Parquet fixed value has length {}, expected {}", + value.size, value_width); + } + } + return Status::OK(); + } + Status _decode_slices(size_t num_values) { _values.resize(num_values); int decoded_count = 0; @@ -413,10 +498,17 @@ Status DeltaBitPackDecoder::_init_header() { "The number of values in a miniblock must be multiple of 32, but it's " + std::to_string(_values_per_mini_block)); } + // Encoded counts are external ULEB32 values. Bound them by the page's advertised logical + // values before any vector uses the count; optional levels may only reduce this upper bound. + if (UNLIKELY(_total_value_count > _expected_values)) { + return Status::Corruption("Parquet delta header advertises {} values, page allows {}", + _total_value_count, _expected_values); + } _total_values_remaining = _total_value_count; - _delta_bit_widths.resize(_mini_blocks_per_block); + _delta_bit_widths.clear(); // init as empty property _block_initialized = false; + _delta_bit_width = 0; _values_remaining_current_mini_block = 0; return Status::OK(); } @@ -428,6 +520,14 @@ Status DeltaBitPackDecoder::_init_block() { return Status::IOError("Init block eof"); } + // One byte follows for each miniblock. Defer allocation until a block is actually consumed so + // a one-value page cannot turn an irrelevant malicious miniblock count into a large resize. + if (UNLIKELY(_mini_blocks_per_block > static_cast(_bit_reader->bytes_left()))) { + return Status::Corruption("Parquet delta miniblock count {} exceeds remaining {} bytes", + _mini_blocks_per_block, _bit_reader->bytes_left()); + } + _delta_bit_widths.resize(_mini_blocks_per_block); + // read the bitwidth of each miniblock uint8_t* bit_width_data = _delta_bit_widths.data(); for (uint32_t i = 0; i < _mini_blocks_per_block; ++i) { diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h index d08a9146ac5cc7..cd59e22785f9df 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -45,6 +45,13 @@ class FixLengthPlainDecoder final : public Decoder { Status skip_values(size_t num_values) override; + void release_scratch(size_t max_retained_bytes) override { + release_vector_if_oversized(&_selected_values, max_retained_bytes); + } + size_t retained_scratch_bytes() const override { + return _selected_values.capacity() * sizeof(uint8_t); + } + private: std::vector _selected_values; }; diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp index 40e435b9b55141..77a99deea172e5 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -37,6 +37,8 @@ Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level _bit_width = cast_set(BitUtil::log2(max_level + 1)); _max_level = max_level; _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; switch (encoding) { case tparquet::Encoding::RLE: { if (slice->size < V1_LEVEL_SIZE) { @@ -48,7 +50,7 @@ Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level if (num_bytes > slice->size - V1_LEVEL_SIZE) { return Status::Corruption("Wrong parquet level format"); } - _rle_decoder = RleDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); + _rle_decoder = RleBatchDecoder(data + V1_LEVEL_SIZE, num_bytes, _bit_width); slice->data += V1_LEVEL_SIZE + num_bytes; slice->size -= V1_LEVEL_SIZE + num_bytes; @@ -77,31 +79,161 @@ Status LevelDecoder::init_v2(const Slice& levels, level_t max_level, uint32_t nu _bit_width = cast_set(BitUtil::log2(max_level + 1)); _max_level = max_level; _num_levels = num_levels; + _has_buffered_level = false; + _can_rewind = false; size_t byte_length = levels.size; - _rle_decoder = - RleDecoder((uint8_t*)levels.data, cast_set(byte_length), _bit_width); + _rle_decoder = RleBatchDecoder((uint8_t*)levels.data, cast_set(byte_length), + _bit_width); return Status::OK(); } size_t LevelDecoder::get_levels(level_t* levels, size_t n) { + _can_rewind = false; // toto template. if (_encoding == tparquet::Encoding::RLE) { n = std::min((size_t)_num_levels, n); - auto num_decoded = _rle_decoder.get_values(levels, n); + size_t num_decoded = 0; + if (_has_buffered_level && n > 0) { + levels[num_decoded++] = _buffered_level; + _has_buffered_level = false; + } + if (num_decoded < n) { + const size_t remaining = n - num_decoded; + _rle_scratch.resize(remaining); + const size_t batch_decoded = + _rle_decoder.GetBatch(_rle_scratch.data(), cast_set(remaining)); + for (size_t i = 0; i < batch_decoded; ++i) { + levels[num_decoded + i] = cast_set(_rle_scratch[i]); + } + num_decoded += batch_decoded; + } _num_levels -= num_decoded; + _can_rewind = false; return num_decoded; } else if (_encoding == tparquet::Encoding::BIT_PACKED) { n = std::min((size_t)_num_levels, n); - for (size_t i = 0; i < n; ++i) { - if (!_bit_packed_decoder.GetValue(_bit_width, &levels[i])) { - throw doris::Exception(ErrorCode::INTERNAL_ERROR, - "Failed to decode BIT_PACKED levels"); + size_t decoded = 0; + for (; decoded < n; ++decoded) { + if (!_bit_packed_decoder.GetValue(_bit_width, &levels[decoded])) { + break; } } - _num_levels -= n; - return n; + _num_levels -= decoded; + return decoded; } return 0; } +size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { + DORIS_CHECK(val != nullptr); + _can_rewind = false; + max_run = std::min(max_run, _num_levels); + if (max_run == 0) { + return 0; + } + if (_encoding == tparquet::Encoding::RLE) { + size_t decoded = 0; + if (_has_buffered_level) { + *val = _buffered_level; + _has_buffered_level = false; + decoded = 1; + } else { + uint16_t first = 0; + if (_rle_decoder.GetBatch(&first, 1) != 1) { + return 0; + } + *val = cast_set(first); + decoded = 1; + } + while (decoded < max_run) { + const int32_t repeats = _rle_decoder.NextNumRepeats(); + if (repeats > 0) { + const level_t repeated = cast_set(_rle_decoder.GetRepeatedValue(0)); + if (repeated != *val) break; + const int32_t consume = std::min(repeats, cast_set(max_run - decoded)); + _rle_decoder.GetRepeatedValue(consume); + decoded += consume; + continue; + } + if (_rle_decoder.NextNumLiterals() == 0) break; + uint16_t literal = 0; + if (!_rle_decoder.GetLiteralValues(1, &literal)) break; + const level_t next = cast_set(literal); + if (next != *val) { + // Batch RLE has no physical rewind; retain the one-value lookahead logically. + _buffered_level = next; + _has_buffered_level = true; + break; + } + ++decoded; + } + _num_levels -= decoded; + _can_rewind = false; + return decoded; + } + if (_encoding != tparquet::Encoding::BIT_PACKED || + !_bit_packed_decoder.GetValue(_bit_width, val)) { + return 0; + } + size_t decoded = 1; + while (decoded < max_run) { + level_t next = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &next)) { + break; + } + if (next != *val) { + // The lookahead belongs to the following run, so cursor APIs must leave it unread. + _bit_packed_decoder.Rewind(_bit_width); + break; + } + ++decoded; + } + _num_levels -= decoded; + return decoded; +} + +level_t LevelDecoder::get_next() { + if (_num_levels == 0) { + return -1; + } + level_t next = -1; + bool decoded = false; + if (_encoding == tparquet::Encoding::RLE) { + if (_has_buffered_level) { + next = _buffered_level; + _has_buffered_level = false; + decoded = true; + } else { + uint16_t value = 0; + decoded = _rle_decoder.GetBatch(&value, 1) == 1; + next = cast_set(value); + } + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + decoded = _bit_packed_decoder.GetValue(_bit_width, &next); + } + if (!decoded) { + return -1; + } + --_num_levels; + _last_level = next; + _can_rewind = true; + return next; +} + +void LevelDecoder::rewind_one() { + if (_encoding == tparquet::Encoding::RLE) { + DORIS_CHECK(_can_rewind && !_has_buffered_level); + _buffered_level = _last_level; + _has_buffered_level = true; + } else if (_encoding == tparquet::Encoding::BIT_PACKED) { + DORIS_CHECK(_can_rewind); + _bit_packed_decoder.Rewind(_bit_width); + } else { + return; + } + // Rewinding restores one advertised level as well as its encoded bits. + ++_num_levels; + _can_rewind = false; +} + } // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h index 7d91b10eb7cdde..69cbe02e9f9ddd 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.h +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -21,6 +21,7 @@ #include #include +#include #include "common/status.h" #include "format/parquet/parquet_common.h" @@ -43,27 +44,30 @@ class LevelDecoder { size_t get_levels(level_t* levels, size_t n); - inline size_t get_next_run(level_t* val, size_t max_run) { - return _rle_decoder.GetNextRun(val, max_run); - } + size_t get_next_run(level_t* val, size_t max_run); - inline level_t get_next() { - level_t next = -1; - _rle_decoder.Get(&next); - return next; - } + level_t get_next(); - inline void rewind_one() { _rle_decoder.RewindOne(); } + void rewind_one(); - const RleDecoder& rle_decoder() const { return _rle_decoder; } + void release_scratch(size_t max_retained_bytes) { + if (_rle_scratch.capacity() * sizeof(uint16_t) > max_retained_bytes) { + std::vector().swap(_rle_scratch); + } + } private: tparquet::Encoding::type _encoding; level_t _bit_width = 0; level_t _max_level = 0; uint32_t _num_levels = 0; - RleDecoder _rle_decoder; + RleBatchDecoder _rle_decoder; + std::vector _rle_scratch; BitReader _bit_packed_decoder; + bool _has_buffered_level = false; + bool _can_rewind = false; + level_t _buffered_level = -1; + level_t _last_level = -1; }; -} // namespace doris::format::parquet::native \ No newline at end of file +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp index 16da9912cd4282..1f28a4a3f61ca6 100644 --- a/be/src/format_v2/parquet/reader/native/level_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -42,14 +42,15 @@ class LevelReaderImpl final : public LevelReader::Impl { public: LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, - bool enable_page_cache) + bool enable_page_cache, std::string page_cache_file_key) : _file(std::move(file)), _column_chunk(std::move(column_chunk)), _field(field), _total_rows(total_rows), _max_buffer_size(max_buffer_size), _io_ctx(io_ctx), - _enable_page_cache(enable_page_cache) {} + _enable_page_cache(enable_page_cache), + _page_cache_file_key(std::move(page_cache_file_key)) {} Status init() override { DORIS_CHECK(_file != nullptr); @@ -70,7 +71,7 @@ class LevelReaderImpl final : public LevelReader::Impl { prefetch_buffer_size); _chunk_reader = std::make_unique>( _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, - ParquetPageReadContext(_enable_page_cache)); + ParquetPageReadContext(_enable_page_cache, _page_cache_file_key)); return _chunk_reader->init(); } @@ -153,6 +154,7 @@ class LevelReaderImpl final : public LevelReader::Impl { size_t _max_buffer_size = 0; io::IOContext* _io_ctx = nullptr; bool _enable_page_cache = false; + std::string _page_cache_file_key; size_t _current_row = 0; std::unique_ptr _stream; std::unique_ptr> _chunk_reader; @@ -161,6 +163,7 @@ class LevelReaderImpl final : public LevelReader::Impl { Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, std::unique_ptr* reader) { DORIS_CHECK(reader != nullptr); DORIS_CHECK(field != nullptr); @@ -168,11 +171,11 @@ Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column if (field->repetition_level > 0) { impl = std::make_unique>(std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, io_ctx, - enable_page_cache); + enable_page_cache, page_cache_file_key); } else { impl = std::make_unique>(std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, io_ctx, - enable_page_cache); + enable_page_cache, page_cache_file_key); } RETURN_IF_ERROR(impl->init()); reader->reset(new LevelReader(std::move(impl))); diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h index df5ae7931a021d..b5e929cb1d08cf 100644 --- a/be/src/format_v2/parquet/reader/native/level_reader.h +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -52,6 +52,7 @@ class LevelReader { static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, bool enable_page_cache, + const std::string& page_cache_file_key, std::unique_ptr* reader); ~LevelReader(); diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp index 14496d6e447665..20f585ae021da4 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -42,8 +42,44 @@ struct IOContext; namespace doris::format::parquet::native { static constexpr size_t INIT_PAGE_HEADER_SIZE = 128; -void ParquetPageCacheKeyBuilder::init(const std::string& path, int64_t mtime) { - _file_key_prefix = fmt::format("{}::{}", path, mtime); +template +Status PageReader::_validate_page_header(uint32_t header_size) const { + if (UNLIKELY(_cur_page_header.compressed_page_size < 0 || + _cur_page_header.uncompressed_page_size < 0)) { + return Status::Corruption("Parquet page has a negative compressed or uncompressed size"); + } + if (UNLIKELY(header_size > _end_offset - _offset || + static_cast(_cur_page_header.compressed_page_size) > + _end_offset - _offset - header_size)) { + // Sizes are untrusted signed Thrift fields and must fit the column chunk before arithmetic. + return Status::Corruption("Parquet page payload exceeds its column chunk"); + } + if (_cur_page_header.__isset.data_page_header_v2) { + const auto& v2 = _cur_page_header.data_page_header_v2; + if (UNLIKELY(v2.num_values < 0 || v2.num_rows < 0 || v2.num_nulls < 0 || + v2.repetition_levels_byte_length < 0 || + v2.definition_levels_byte_length < 0)) { + return Status::Corruption("Parquet data page v2 has negative counts or level sizes"); + } + if (UNLIKELY(v2.num_nulls > v2.num_values || v2.num_rows > v2.num_values)) { + return Status::Corruption( + "Parquet data page v2 null or row count exceeds its value count"); + } + const uint64_t level_bytes = static_cast(v2.repetition_levels_byte_length) + + static_cast(v2.definition_levels_byte_length); + if (UNLIKELY(level_bytes > static_cast(_cur_page_header.compressed_page_size) || + level_bytes > + static_cast(_cur_page_header.uncompressed_page_size))) { + return Status::Corruption("Parquet data page v2 level bytes exceed the page payload"); + } + } else if (_cur_page_header.__isset.data_page_header && + UNLIKELY(_cur_page_header.data_page_header.num_values < 0)) { + return Status::Corruption("Parquet data page has a negative value count"); + } else if (_cur_page_header.__isset.dictionary_page_header && + UNLIKELY(_cur_page_header.dictionary_page_header.num_values < 0)) { + return Status::Corruption("Parquet dictionary page has a negative value count"); + } + return Status::OK(); } template @@ -64,10 +100,10 @@ PageReader::PageReader(io::BufferedStreamReader* re _offset_index(offset_index) { _next_header_offset = _offset; _state = INITIALIZED; - _page_cache_key_builder.init(_reader->path(), _reader->mtime()); + _page_cache_key_builder.init(_page_read_ctx.page_cache_file_key); if constexpr (OFFSET_INDEX) { - _end_row = _offset_index->page_locations.size() >= 2 + _end_row = _offset_index != nullptr && _offset_index->page_locations.size() >= 2 ? _offset_index->page_locations[1].first_row_index : _total_rows; } @@ -114,6 +150,11 @@ Status PageReader::parse_page_header() { auto st = deserialize_thrift_msg(reinterpret_cast(s.data), &real_header_size, true, &_cur_page_header); if (!st.ok()) return st; + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } // Increment page cache counters for a true cache hit on header+payload _page_statistics.page_cache_hit_counter += 1; // Detect whether the cached payload is compressed or decompressed and record @@ -180,6 +221,7 @@ Status PageReader::parse_page_header() { } if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_validate_page_header(real_header_size)); if (is_header_v2()) { _end_row = _start_row + _cur_page_header.data_page_header_v2.num_rows; } else if constexpr (!IN_COLLECTION) { @@ -187,6 +229,15 @@ Status PageReader::parse_page_header() { } } + if constexpr (OFFSET_INDEX == true) { + RETURN_IF_ERROR(_validate_page_header(real_header_size)); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE || + _cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { + ++_page_statistics.data_page_read_counter; + } + // Save header bytes for possible cache insertion later _header_buf.assign(page_header_buf, page_header_buf + real_header_size); _last_header_size = real_header_size; @@ -217,4 +268,4 @@ template class PageReader; template class PageReader; template class PageReader; -} // namespace doris::format::parquet::native \ No newline at end of file +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h index fe4dc52a2b69c3..641a047eb38ca8 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.h +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -54,14 +54,23 @@ namespace doris::format::parquet::native { // Session-level options for parquet page reading/caching. struct ParquetPageReadContext { - bool enable_parquet_file_page_cache = true; + // A default-constructed context has no stable file identity, so cache lookup must stay off. + bool enable_parquet_file_page_cache = false; + std::string page_cache_file_key; ParquetPageReadContext() = default; - ParquetPageReadContext(bool enable_parquet_file_page_cache) - : enable_parquet_file_page_cache(enable_parquet_file_page_cache) {} + ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key) + : enable_parquet_file_page_cache(enable_parquet_file_page_cache && + !page_cache_file_key.empty()), + page_cache_file_key(std::move(page_cache_file_key)) {} }; inline bool should_cache_decompressed(const tparquet::PageHeader* header, const tparquet::ColumnMetaData& metadata) { + // Data Page V2 declares its payload representation independently of the column codec. A warm + // hit must never send an explicitly uncompressed cached payload through the codec again. + if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed) { + return true; + } if (header->compressed_page_size <= 0) return true; if (metadata.codec == tparquet::CompressionCodec::UNCOMPRESSED) return true; if (header->uncompressed_page_size == 0) return true; @@ -73,7 +82,7 @@ inline bool should_cache_decompressed(const tparquet::PageHeader* header, class ParquetPageCacheKeyBuilder { public: - void init(const std::string& path, int64_t mtime); + void init(std::string file_key) { _file_key_prefix = std::move(file_key); } StoragePageCache::CacheKey make_key(uint64_t end_offset, int64_t offset) const { return StoragePageCache::CacheKey(_file_key_prefix, end_offset, offset); } @@ -98,6 +107,7 @@ class PageReader { int64_t page_cache_compressed_write_counter = 0; int64_t page_cache_decompressed_write_counter = 0; int64_t page_read_counter = 0; + int64_t data_page_read_counter = 0; }; PageReader(io::BufferedStreamReader* reader, io::IOContext* io_ctx, uint64_t offset, @@ -108,7 +118,8 @@ class PageReader { bool has_next_page() const { if constexpr (OFFSET_INDEX) { - return _page_index + 1 != _offset_index->page_locations.size(); + return _offset_index != nullptr && + _page_index + 1 < _offset_index->page_locations.size(); } else { // Deprecated // Parquet file may not be standardized, @@ -126,6 +137,10 @@ class PageReader { Status next_page() { _page_statistics.skip_page_header_num += _state == INITIALIZED; if constexpr (OFFSET_INDEX) { + if (UNLIKELY(_offset_index == nullptr || + _page_index + 1 >= _offset_index->page_locations.size())) { + return Status::Corruption("Parquet OffsetIndex has no next page location"); + } _page_index++; _start_row = _offset_index->page_locations[_page_index].first_row_index; if (_page_index + 1 < _offset_index->page_locations.size()) { @@ -212,6 +227,8 @@ class PageReader { } private: + Status _validate_page_header(uint32_t header_size) const; + enum PageReaderState { INITIALIZED, HEADER_PARSED, DATA_LOADED }; PageReaderState _state = INITIALIZED; PageStatistics _page_statistics; @@ -258,4 +275,4 @@ std::unique_ptr> create_page_reader( reader, io_ctx, offset, length, total_rows, metadata, ctx, offset_index); } -} // namespace doris::format::parquet::native \ No newline at end of file +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index ebb92202e59aa4..f100cd780c1f38 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -159,7 +159,8 @@ Status NativeColumnReader::create( const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, - bool enable_page_cache, bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter, ParquetColumnReaderProfile profile, std::unique_ptr* reader) { if (reader == nullptr) { return Status::InvalidArgument("Native parquet reader result is null"); @@ -197,7 +198,7 @@ Status NativeColumnReader::create( RETURN_IF_ERROR(native_reader->init( std::move(file), metadata, row_group_id, field, std::move(schema_node), std::move(projected_ids), selected_ranges, offset_indexes, timezone, io_ctx, - runtime_state, enable_page_cache, enable_dictionary_filter)); + runtime_state, enable_page_cache, page_cache_file_key, enable_dictionary_filter)); *reader = std::move(native_reader); return Status::OK(); } @@ -208,7 +209,8 @@ Status NativeColumnReader::init( std::set projected_column_ids, const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, - bool enable_page_cache, bool enable_dictionary_filter) { + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter) { DORIS_CHECK(file != nullptr); DORIS_CHECK(metadata != nullptr); DORIS_CHECK(field != nullptr); @@ -224,7 +226,9 @@ Status NativeColumnReader::init( DORIS_CHECK(range.start + range.length <= _row_group_rows); _row_ranges.add(::doris::RowRange(range.start, range.start + range.length)); } - _offset_indexes = offset_indexes; + // Offset indexes are immutable row-group metadata owned by ParquetScanScheduler. Sharing them + // avoids retaining the full N-column page-location map once per projected reader. + _offset_indexes = &offset_indexes; _schema_node = std::move(schema_node); _projected_column_ids = std::move(projected_column_ids); _dictionary_filter_enabled = enable_dictionary_filter; @@ -243,10 +247,10 @@ Status NativeColumnReader::init( _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); native_runtime_state = _page_cache_runtime_state.get(); } - RETURN_IF_ERROR(native::ColumnReader::create(std::move(file), field, row_group, _row_ranges, - timezone, io_ctx, _native_reader, max_buffer_size, - _offset_indexes, native_runtime_state, false, - _projected_column_ids, _filter_column_ids)); + RETURN_IF_ERROR(native::ColumnReader::create( + std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, + max_buffer_size, *_offset_indexes, native_runtime_state, false, _projected_column_ids, + _filter_column_ids, page_cache_file_key)); DORIS_CHECK(_native_reader != nullptr); _skip_column = _type->create_column(); return Status::OK(); @@ -346,7 +350,8 @@ Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); advance_selected_span(*rows_read); update_reader_read_rows(*rows_read); - record_page_fragments(sync_native_profile()); + int64_t max_leaf_page_reads = 0; + record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } @@ -377,7 +382,9 @@ Status NativeColumnReader::skip(int64_t rows) { const int64_t selected_rows = std::min(remaining, range.start + range.length - _logical_row_position); _skip_column->clear(); - _filter_scratch.resize(static_cast(selected_rows)); + // resize() preserves survivor bytes from the previous select(). An all-filtered nested read + // consumes the raw bitmap, so every slot must be explicitly reset before a lazy skip. + _filter_scratch.assign(static_cast(selected_rows), 0); int64_t rows_read = 0; RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, _type, false, &rows_read)); @@ -388,7 +395,8 @@ Status NativeColumnReader::skip(int64_t rows) { remaining -= rows_read; } update_reader_skip_rows(native_skipped_rows); - record_page_fragments(sync_native_profile()); + int64_t max_leaf_page_reads = 0; + record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } @@ -415,7 +423,8 @@ Status NativeColumnReader::select(const SelectionVector& selection, uint16_t sel } update_reader_read_rows(selected_rows); update_reader_skip_rows(batch_rows - selected_rows); - record_page_fragments(sync_native_profile()); + int64_t max_leaf_page_reads = 0; + record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } @@ -505,20 +514,23 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& } update_reader_read_rows(cast_set(matched_ids.size())); update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); - record_page_fragments(sync_native_profile()); + int64_t max_leaf_page_reads = 0; + record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } -void NativeColumnReader::record_page_fragments(int64_t page_fragments) { +void NativeColumnReader::record_page_fragments(int64_t page_fragments, + int64_t max_leaf_page_reads) { if (_profile.native_page_fragments != nullptr) { COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); } - if (page_fragments > 1 && _profile.page_crossing_batches != nullptr) { + // Summed fragments from MAP/STRUCT siblings do not mean any individual leaf crossed a page. + if (max_leaf_page_reads > 1 && _profile.page_crossing_batches != nullptr) { COUNTER_UPDATE(_profile.page_crossing_batches, 1); } } -int64_t NativeColumnReader::sync_native_profile() { +int64_t NativeColumnReader::sync_native_profile(int64_t* max_leaf_page_reads) { if (_native_reader == nullptr) { return 0; } @@ -587,6 +599,22 @@ int64_t NativeColumnReader::sync_native_profile() { stats.read_page_header_time - reported.read_page_header_time); } const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; + int64_t max_leaf_delta = 0; + if (stats.leaf_page_read_counters.size() == reported.leaf_page_read_counters.size()) { + for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { + max_leaf_delta = + std::max(max_leaf_delta, stats.leaf_page_read_counters[leaf] - + reported.leaf_page_read_counters[leaf]); + } + } else { + // The tree shape is stable after initialization; retain a safe first-snapshot fallback. + for (const int64_t page_reads : stats.leaf_page_read_counters) { + max_leaf_delta = std::max(max_leaf_delta, page_reads); + } + } + if (max_leaf_page_reads != nullptr) { + *max_leaf_page_reads = max_leaf_delta; + } if (_profile.page_read_count != nullptr) { COUNTER_UPDATE(_profile.page_read_count, page_read_delta); } diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index c642d48d7dd0ea..3fe6e6e91ff338 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -63,7 +63,8 @@ class NativeColumnReader final : public ParquetColumnReader { const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, bool enable_page_cache, - bool enable_dictionary_filter, ParquetColumnReaderProfile profile, + const std::string& page_cache_file_key, bool enable_dictionary_filter, + ParquetColumnReaderProfile profile, std::unique_ptr* reader); ~NativeColumnReader() override; @@ -88,13 +89,14 @@ class NativeColumnReader final : public ParquetColumnReader { const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, - bool enable_page_cache, bool enable_dictionary_filter); + bool enable_page_cache, const std::string& page_cache_file_key, + bool enable_dictionary_filter); Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, MutableColumnPtr& column, const DataTypePtr& output_type, bool dictionary_ids, int64_t* rows_read); - int64_t sync_native_profile(); - void record_page_fragments(int64_t page_fragments); + int64_t sync_native_profile(int64_t* max_leaf_page_reads = nullptr); + void record_page_fragments(int64_t page_fragments, int64_t max_leaf_page_reads); Status validate_selected_span(int64_t rows); void advance_selected_span(int64_t rows); @@ -102,7 +104,7 @@ class NativeColumnReader final : public ParquetColumnReader { ::doris::RowRanges _row_ranges; std::set _projected_column_ids; std::set _filter_column_ids; - std::unordered_map _offset_indexes; + const std::unordered_map* _offset_indexes = nullptr; std::shared_ptr _schema_node; std::unique_ptr _native_reader; std::unique_ptr _page_cache_runtime_state; diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index ef13f585fb6a6d..09de209f3bc66e 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -167,6 +167,7 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( } Status IcebergPositionDeleteSysTableV2Reader::get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); DORIS_CHECK(block != nullptr); DORIS_CHECK(eos != nullptr); diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index c67cece6e05b8c..6c38147f1eccdd 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -37,6 +37,7 @@ #include "format/arrow/arrow_utils.h" #include "format_v2/materialized_reader_util.h" #include "runtime/descriptors.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" #include "util/timezone_utils.h" @@ -177,7 +178,27 @@ RemoteDorisFileReader::~RemoteDorisFileReader() { static_cast(close()); } +void RemoteDorisFileReader::_init_profile() { + if (_profile == nullptr) { + return; + } + const auto hierarchy = file_scan_profile::ensure_hierarchy(_profile); + _io_time = hierarchy.io; + static const char* remote_profile = "RemoteDorisFileReader"; + _total_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, remote_profile, file_scan_profile::FILE_READER, 1); + _open_stream_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisOpenStreamTime", remote_profile, 1); + _next_batch_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisNextBatchTime", remote_profile, 1); + _materialize_time = + ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisMaterializeTime", remote_profile, 1); + _filter_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "RemoteDorisFilterTime", remote_profile, 1); +} + Status RemoteDorisFileReader::init(RuntimeState* state) { + _init_profile(); + SCOPED_TIMER(_total_time); (void)state; RETURN_IF_ERROR(validate_remote_doris_range(_range)); RETURN_IF_ERROR(_build_col_name_to_file_id()); @@ -186,6 +207,7 @@ Status RemoteDorisFileReader::init(RuntimeState* state) { } Status RemoteDorisFileReader::get_schema(std::vector* file_schema) const { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_schema != nullptr); file_schema->clear(); file_schema->reserve(_file_slot_descs.size()); @@ -206,6 +228,8 @@ Status RemoteDorisFileReader::get_schema(std::vector* file_sch } Status RemoteDorisFileReader::open(std::shared_ptr request) { + SCOPED_TIMER(_total_time); + SCOPED_TIMER(_open_stream_time); RETURN_IF_ERROR(FileReader::open(std::move(request))); RETURN_IF_ERROR(_open_stream()); _eof = false; @@ -213,6 +237,7 @@ Status RemoteDorisFileReader::open(std::shared_ptr request) { } Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* eof) { + SCOPED_TIMER(_total_time); DORIS_CHECK(file_block != nullptr); DORIS_CHECK(rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -223,21 +248,32 @@ Status RemoteDorisFileReader::get_block(Block* file_block, size_t* rows, bool* e *rows = 0; *eof = false; std::shared_ptr batch; - RETURN_IF_ERROR(_stream->next(&batch)); + { + SCOPED_TIMER(_io_time); + SCOPED_TIMER(_next_batch_time); + RETURN_IF_ERROR(_stream->next(&batch)); + } if (batch == nullptr) { *eof = true; _eof = true; return Status::OK(); } - RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + { + SCOPED_TIMER(_materialize_time); + RETURN_IF_ERROR(_materialize_record_batch(*batch, file_block, rows)); + } _record_scan_rows(cast_set(*rows)); - RETURN_IF_ERROR( - apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + { + SCOPED_TIMER(_filter_time); + RETURN_IF_ERROR( + apply_materialized_reader_filters(_request.get(), _io_ctx.get(), file_block, rows)); + } return Status::OK(); } Status RemoteDorisFileReader::close() { + SCOPED_TIMER(_total_time); if (_stream != nullptr) { RETURN_IF_ERROR(_stream->close()); _stream.reset(); diff --git a/be/src/format_v2/table/remote_doris_reader.h b/be/src/format_v2/table/remote_doris_reader.h index b4dd2a505a95ad..79c37a29be4a06 100644 --- a/be/src/format_v2/table/remote_doris_reader.h +++ b/be/src/format_v2/table/remote_doris_reader.h @@ -71,6 +71,7 @@ class RemoteDorisFileReader final : public FileReader { Status close() override; private: + void _init_profile() override; Status _open_stream(); Status _materialize_record_batch(const arrow::RecordBatch& batch, Block* file_block, size_t* rows) const; @@ -83,6 +84,12 @@ class RemoteDorisFileReader final : public FileReader { const std::vector _file_slot_descs; RemoteDorisStreamFactory _stream_factory; cctz::time_zone _ctz; + RuntimeProfile::Counter* _total_time = nullptr; + RuntimeProfile::Counter* _open_stream_time = nullptr; + RuntimeProfile::Counter* _next_batch_time = nullptr; + RuntimeProfile::Counter* _io_time = nullptr; + RuntimeProfile::Counter* _materialize_time = nullptr; + RuntimeProfile::Counter* _filter_time = nullptr; std::unique_ptr _stream; std::unordered_map _col_name_to_file_id; }; diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index e8725ed65fa49b..77a808e298bc69 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -49,6 +49,7 @@ #include "format_v2/native/native_reader.h" #include "format_v2/orc/orc_reader.h" #include "format_v2/parquet/parquet_reader.h" +#include "runtime/file_scan_profile.h" #include "storage/segment/condition_cache.h" #include "util/debug_points.h" #include "util/string_util.h" @@ -536,8 +537,13 @@ Status TableReader::init(TableReadOptions&& options) { _conjuncts = std::move(options.conjuncts); if (_scanner_profile != nullptr) { - static const char* table_profile = "TableReader"; - ADD_TIMER_WITH_LEVEL(_scanner_profile, table_profile, 1); + const auto hierarchy = file_scan_profile::ensure_hierarchy(_scanner_profile); + static const char* table_profile = file_scan_profile::TABLE_READER; + static const char* file_reader_profile = file_scan_profile::FILE_READER; + _profile.total_timer = hierarchy.table_reader; + _profile.file_reader_total_timer = hierarchy.file_reader; + _profile.init_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "InitTime", table_profile, 1); _profile.num_delete_files = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteFiles", TUnit::UNIT, table_profile, 1); _profile.num_delete_rows = ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, "NumDeleteRows", @@ -570,11 +576,32 @@ Status TableReader::init(TableReadOptions&& options) { ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "PushDownAggTime", table_profile, 1); _profile.open_reader_timer = ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "OpenReaderTime", table_profile, 1); - _profile.runtime_filter_partition_prune_timer = ADD_TIMER_WITH_LEVEL( - _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", 1); - _profile.runtime_filter_partition_pruned_range_counter = ADD_COUNTER_WITH_LEVEL( - _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, 1); - } + _profile.runtime_filter_partition_prune_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileScannerRuntimeFilterPartitionPruningTime", table_profile, 1); + _profile.runtime_filter_partition_pruned_range_counter = ADD_CHILD_COUNTER_WITH_LEVEL( + _scanner_profile, "RuntimeFilterPartitionPrunedRangeNum", TUnit::UNIT, + table_profile, 1); + _profile.close_timer = + ADD_CHILD_TIMER_WITH_LEVEL(_scanner_profile, "CloseTime", table_profile, 1); + // Lifecycle timer names remain globally unique because RuntimeProfile's visual hierarchy + // does not namespace counters that share the same display parent. + _profile.file_reader_init_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderInitTime", file_reader_profile, 1); + _profile.file_reader_schema_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetSchemaTime", file_reader_profile, 1); + _profile.file_reader_mapper_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCreateColumnMapperTime", file_reader_profile, 1); + _profile.file_reader_open_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderOpenTime", file_reader_profile, 1); + _profile.file_reader_get_block_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderGetBlockTime", file_reader_profile, 1); + _profile.file_reader_aggregate_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderAggregatePushDownTime", file_reader_profile, 1); + _profile.file_reader_close_timer = ADD_CHILD_TIMER_WITH_LEVEL( + _scanner_profile, "FileReaderCloseTime", file_reader_profile, 1); + } + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); return Status::OK(); } @@ -723,7 +750,12 @@ Status TableReader::create_next_reader(bool* eos) { if (_batch_size > 0) { _data_reader.reader->set_batch_size(_batch_size); } - Status st = _data_reader.reader->init(_runtime_state); + Status st; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_init_timer); + st = _data_reader.reader->init(_runtime_state); + } if (!st.ok()) { if (_io_ctx != nullptr && _io_ctx->should_stop && st.is()) { *eos = true; @@ -828,6 +860,7 @@ std::unique_ptr create_file_description(const TFileRangeDes } Status TableReader::prepare_split(const SplitReadOptions& options) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.prepare_split_timer); _current_split_pruned = false; _all_runtime_filters_applied_for_split = options.all_runtime_filters_applied; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index cddddccd6bd3e1..dadd528e243e70 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -99,6 +99,8 @@ struct ProjectedColumnBuildContext { }; struct ReadProfile { + RuntimeProfile::Counter* total_timer = nullptr; + RuntimeProfile::Counter* init_timer = nullptr; RuntimeProfile::Counter* num_delete_files = nullptr; RuntimeProfile::Counter* num_delete_rows = nullptr; RuntimeProfile::Counter* parse_delete_file_time = nullptr; @@ -115,6 +117,15 @@ struct ReadProfile { RuntimeProfile::Counter* open_reader_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_prune_timer = nullptr; RuntimeProfile::Counter* runtime_filter_partition_pruned_range_counter = nullptr; + RuntimeProfile::Counter* close_timer = nullptr; + RuntimeProfile::Counter* file_reader_total_timer = nullptr; + RuntimeProfile::Counter* file_reader_init_timer = nullptr; + RuntimeProfile::Counter* file_reader_schema_timer = nullptr; + RuntimeProfile::Counter* file_reader_mapper_timer = nullptr; + RuntimeProfile::Counter* file_reader_open_timer = nullptr; + RuntimeProfile::Counter* file_reader_get_block_timer = nullptr; + RuntimeProfile::Counter* file_reader_aggregate_timer = nullptr; + RuntimeProfile::Counter* file_reader_close_timer = nullptr; }; struct TableReadOptions { @@ -236,6 +247,7 @@ class TableReader { // advances across EOF, and closes exhausted readers. Subclasses provide protected hooks for // table-format-specific behavior. virtual Status get_block(Block* block, bool* eos) { + SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.exec_timer); DORIS_CHECK(block->columns() == _projected_columns.size()); block->clear_column_data(_projected_columns.size()); @@ -285,8 +297,12 @@ class TableReader { _data_reader.block_template.clear_column_data( cast_set(_data_reader.file_block_layout.size())); size_t current_rows = 0; - RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, - ¤t_rows, ¤t_eof)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_get_block_timer); + RETURN_IF_ERROR(_data_reader.reader->get_block(&_data_reader.block_template, + ¤t_rows, ¤t_eof)); + } const bool stopped_during_read = _io_ctx != nullptr && _io_ctx->should_stop; if (current_rows == 0) { if (current_eof) { @@ -317,6 +333,8 @@ class TableReader { // Close the table reader and the currently active file reader. Subclasses that hold additional // table-format resources should override this and call TableReader::close() first. virtual Status close() { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader) { RETURN_IF_ERROR(close_current_reader()); } @@ -379,14 +397,22 @@ class TableReader { SCOPED_TIMER(_profile.open_reader_timer); // 1. Get file schema and create column mapping. std::vector file_schema; - RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_schema_timer); + RETURN_IF_ERROR(_data_reader.reader->get_schema(&file_schema)); + } // For Paimon/Hudi, FE can provide field ids through `history_schema_info`. Annotate the // file schema before column mapping when the table format maps columns by field id. RETURN_IF_ERROR(annotate_file_schema(&file_schema)); _data_reader.file_schema = file_schema; _mapper_options.mode = mapping_mode(); - _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_mapper_timer); + _data_reader.column_mapper = _data_reader.reader->create_column_mapper(_mapper_options); + } DORIS_CHECK(_data_reader.column_mapper != nullptr); RETURN_IF_ERROR(_data_reader.column_mapper->create_mapping(_projected_columns, _partition_values, file_schema)); @@ -472,7 +498,11 @@ class TableReader { VLOG_DEBUG << "TableReader debug: " << debug_string(); } RETURN_IF_ERROR(_open_mapping_exprs()); - RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_open_timer); + RETURN_IF_ERROR(_data_reader.reader->open(file_request)); + } RETURN_IF_ERROR(_init_reader_condition_cache(*file_request)); return Status::OK(); } @@ -679,7 +709,11 @@ class TableReader { // close(), so it should remain idempotent. virtual Status close_current_reader() { _finalize_reader_condition_cache(); - RETURN_IF_ERROR(_data_reader.reader->close()); + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_close_timer); + RETURN_IF_ERROR(_data_reader.reader->close()); + } _data_reader.reader.reset(); if (_data_reader.column_mapper != nullptr) { _data_reader.column_mapper->clear(); @@ -960,7 +994,12 @@ class TableReader { FileAggregateRequest file_request; RETURN_IF_ERROR(_build_file_aggregate_request(_push_down_agg_type, &file_request)); FileAggregateResult file_result; - const auto status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + Status status; + { + SCOPED_TIMER(_profile.file_reader_total_timer); + SCOPED_TIMER(_profile.file_reader_aggregate_timer); + status = _data_reader.reader->get_aggregate_result(file_request, &file_result); + } if (status.is()) { return Status::OK(); } diff --git a/be/src/io/cache/block_file_cache_profile.cpp b/be/src/io/cache/block_file_cache_profile.cpp index 751d422f044c1d..7aed33ccaa6e07 100644 --- a/be/src/io/cache/block_file_cache_profile.cpp +++ b/be/src/io/cache/block_file_cache_profile.cpp @@ -126,13 +126,26 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren SUBTRACT_FIELD(segment_footer_index_bytes_write_into_cache); SUBTRACT_FIELD(remote_only_on_miss_triggered); SUBTRACT_FIELD(remote_only_on_miss_threshold_bytes); + + SUBTRACT_FIELD(num_cross_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_cross_cg_peer); + SUBTRACT_FIELD(cross_cg_peer_io_timer); + SUBTRACT_FIELD(num_same_cg_peer_io_total); + SUBTRACT_FIELD(bytes_read_from_same_cg_peer); + SUBTRACT_FIELD(same_cg_peer_io_timer); + SUBTRACT_FIELD(num_peer_race_peer_win); + SUBTRACT_FIELD(num_peer_race_s3_win); + SUBTRACT_FIELD(num_peer_lazy_fetch); + SUBTRACT_FIELD(peer_lazy_fetch_timer); #undef SUBTRACT_FIELD return diff; } -FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _profile(profile) { +FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile, + const std::string& parent_counter) + : _profile(profile) { static const char* cache_profile = "FileCache"; - ADD_TIMER_WITH_LEVEL(profile, cache_profile, 2); + total_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, cache_profile, parent_counter.c_str(), 2); num_local_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumLocalIOTotal", TUnit::UNIT, cache_profile, 1); num_remote_io_total = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NumRemoteIOTotal", TUnit::UNIT, @@ -239,6 +252,12 @@ FileCacheProfileReporter::FileCacheProfileReporter(RuntimeProfile* profile) : _p } void FileCacheProfileReporter::update(const FileCacheStatistics* statistics) const { + // These are the outer cache-path phases. Their sum keeps the group timer actionable instead of + // displaying zero while individual cache IO and coordination timers are non-zero. + COUNTER_UPDATE(total_time, statistics->local_io_timer + statistics->remote_io_timer + + statistics->peer_io_timer + statistics->remote_wait_timer + + statistics->write_cache_io_timer + + statistics->cache_get_or_set_timer); COUNTER_UPDATE(num_local_io_total, statistics->num_local_io_total); COUNTER_UPDATE(num_remote_io_total, statistics->num_remote_io_total); COUNTER_UPDATE(num_peer_io_total, statistics->num_peer_io_total); diff --git a/be/src/io/cache/block_file_cache_profile.h b/be/src/io/cache/block_file_cache_profile.h index 2a6e7b33980bb2..d3fd31033649c8 100644 --- a/be/src/io/cache/block_file_cache_profile.h +++ b/be/src/io/cache/block_file_cache_profile.h @@ -71,6 +71,7 @@ FileCacheStatistics diff_file_cache_statistics(const FileCacheStatistics& curren struct FileCacheProfileReporter { RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* total_time = nullptr; RuntimeProfile::Counter* num_local_io_total = nullptr; RuntimeProfile::Counter* num_remote_io_total = nullptr; RuntimeProfile::Counter* num_peer_io_total = nullptr; @@ -129,7 +130,9 @@ struct FileCacheProfileReporter { RuntimeProfile::Counter* num_peer_lazy_fetch = nullptr; RuntimeProfile::Counter* peer_lazy_fetch_timer = nullptr; - FileCacheProfileReporter(RuntimeProfile* profile); + explicit FileCacheProfileReporter( + RuntimeProfile* profile, + const std::string& parent_counter = RuntimeProfile::ROOT_COUNTER); void update(const FileCacheStatistics* statistics) const; }; diff --git a/be/src/io/fs/buffered_reader.cpp b/be/src/io/fs/buffered_reader.cpp index 246ee5e480db5d..386c3e4192c3d6 100644 --- a/be/src/io/fs/buffered_reader.cpp +++ b/be/src/io/fs/buffered_reader.cpp @@ -32,6 +32,7 @@ #include "common/status.h" #include "core/custom_allocator.h" #include "runtime/exec_env.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -655,7 +656,9 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File std::function sync_buffer = nullptr; if (profile != nullptr) { const char* prefetch_buffered_reader = "PrefetchBufferedReader"; - ADD_TIMER(profile, prefetch_buffered_reader); + auto* total_time = + ADD_CHILD_TIMER(profile, prefetch_buffered_reader, + file_scan_profile::parent_or_root(profile, file_scan_profile::IO)); auto copy_time = ADD_CHILD_TIMER(profile, "CopyTime", prefetch_buffered_reader); auto read_time = ADD_CHILD_TIMER(profile, "ReadTime", prefetch_buffered_reader); auto prefetch_request_io = @@ -667,6 +670,7 @@ PrefetchBufferedReader::PrefetchBufferedReader(RuntimeProfile* profile, io::File auto request_bytes = ADD_CHILD_COUNTER(profile, "RequestBytes", TUnit::BYTES, prefetch_buffered_reader); sync_buffer = [=](PrefetchBuffer& buf) { + COUNTER_UPDATE(total_time, buf._statis.copy_time + buf._statis.read_time); COUNTER_UPDATE(copy_time, buf._statis.copy_time); COUNTER_UPDATE(read_time, buf._statis.read_time); COUNTER_UPDATE(prefetch_request_io, buf._statis.prefetch_request_io); @@ -924,7 +928,9 @@ RangeCacheFileReader::RangeCacheFileReader(RuntimeProfile* profile, io::FileRead if (_profile != nullptr) { const char* random_profile = "RangeCacheFileReader"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, random_profile, 1); _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, @@ -985,6 +991,7 @@ Status RangeCacheFileReader::read_at_impl(size_t offset, Slice result, size_t* b void RangeCacheFileReader::_collect_profile_before_close() { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _cache_statistics.request_time); COUNTER_UPDATE(_request_io, _cache_statistics.request_io); COUNTER_UPDATE(_request_bytes, _cache_statistics.request_bytes); COUNTER_UPDATE(_request_time, _cache_statistics.request_time); diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h index beaf3e87a8d3ce..9b0e66ebbd078b 100644 --- a/be/src/io/fs/buffered_reader.h +++ b/be/src/io/fs/buffered_reader.h @@ -38,6 +38,7 @@ #include "io/fs/path.h" #include "io/fs/s3_file_reader.h" #include "io/io_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "storage/olap_define.h" #include "util/slice.h" @@ -179,6 +180,7 @@ class RangeCacheFileReader : public io::FileReader { bool _closed = false; RuntimeProfile::Counter* _request_io = nullptr; + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _request_bytes = nullptr; RuntimeProfile::Counter* _request_time = nullptr; RuntimeProfile::Counter* _read_to_cache_time = nullptr; @@ -301,7 +303,9 @@ class MergeRangeFileReader : public io::FileReader { if (_profile != nullptr) { const char* random_profile = "MergedSmallIO"; - ADD_TIMER_WITH_LEVEL(_profile, random_profile, 1); + _total_time = ADD_CHILD_TIMER_WITH_LEVEL( + _profile, random_profile, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", random_profile, 1); _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", random_profile, 1); _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, @@ -350,6 +354,7 @@ class MergeRangeFileReader : public io::FileReader { void _collect_profile_before_close() override { if (_profile != nullptr) { + COUNTER_UPDATE(_total_time, _statistics.copy_time + _statistics.read_time); COUNTER_UPDATE(_copy_time, _statistics.copy_time); COUNTER_UPDATE(_read_time, _statistics.read_time); COUNTER_UPDATE(_request_io, _statistics.request_io); @@ -363,6 +368,7 @@ class MergeRangeFileReader : public io::FileReader { } private: + RuntimeProfile::Counter* _total_time = nullptr; RuntimeProfile::Counter* _copy_time = nullptr; RuntimeProfile::Counter* _read_time = nullptr; RuntimeProfile::Counter* _request_io = nullptr; diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index f97b2f80cd6ee6..f7d98461035afb 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -40,4 +40,15 @@ std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size); } +std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, + const io::FileDescription& file_description, + bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + auto key = get_key(std::move(file_reader), file_description); + // Schema mapping changes the cached object, so those options are part of its identity. + key.push_back(static_cast(enable_mapping_varbinary)); + key.push_back(static_cast(enable_mapping_timestamp_tz)); + return key; +} + } // namespace doris diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 0c62c963ce122d..054914a9519041 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -41,6 +41,10 @@ class FileMetaCache { static std::string get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description); + static std::string get_key(io::FileReaderSPtr file_reader, + const io::FileDescription& file_description, + bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); + bool lookup(const std::string& key, ObjLRUCache::CacheHandle* handle) { return _cache.lookup({key}, handle); } diff --git a/be/src/io/fs/hdfs_file_reader.cpp b/be/src/io/fs/hdfs_file_reader.cpp index d99db30f1c00fc..6e363636980d19 100644 --- a/be/src/io/fs/hdfs_file_reader.cpp +++ b/be/src/io/fs/hdfs_file_reader.cpp @@ -32,6 +32,7 @@ #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/hdfs_util.h" +#include "runtime/file_scan_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" #include "service/backend_options.h" @@ -83,7 +84,9 @@ HdfsFileReader::HdfsFileReader(Path path, std::string fs_name, FileHandleCache:: if (_profile != nullptr && is_hdfs(_fs_name)) { #ifdef USE_HADOOP_HDFS const char* hdfs_profile_name = "HdfsIO"; - ADD_TIMER(_profile, hdfs_profile_name); + _total_read_time = + ADD_CHILD_TIMER(_profile, hdfs_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); _hdfs_profile.total_bytes_read = ADD_CHILD_COUNTER(_profile, "TotalBytesRead", TUnit::BYTES, hdfs_profile_name); _hdfs_profile.total_local_bytes_read = @@ -117,6 +120,7 @@ Status HdfsFileReader::close() { Status HdfsFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read, const IOContext* io_ctx) { + SCOPED_TIMER(_total_read_time); auto st = do_read_at_impl(offset, result, bytes_read, io_ctx); if (!st.ok()) { _handle = nullptr; diff --git a/be/src/io/fs/hdfs_file_reader.h b/be/src/io/fs/hdfs_file_reader.h index 08f98bca29af0c..7bb73f30909452 100644 --- a/be/src/io/fs/hdfs_file_reader.h +++ b/be/src/io/fs/hdfs_file_reader.h @@ -88,6 +88,7 @@ class HdfsFileReader final : public FileReader { CachedHdfsFileHandle* _handle = nullptr; // owned by _cached_file_handle std::atomic _closed = false; RuntimeProfile* _profile = nullptr; + RuntimeProfile::Counter* _total_read_time = nullptr; int64_t _mtime; #ifdef USE_HADOOP_HDFS HDFSProfile _hdfs_profile; diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index 4eaa10f3311e06..af8dde36d2df50 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -37,6 +37,7 @@ #include "io/fs/err_utils.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_common.h" +#include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" #include "runtime/workload_management/io_throttle.h" @@ -214,7 +215,9 @@ Status S3FileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_rea void S3FileReader::_collect_profile_before_close() { if (_profile != nullptr) { const char* s3_profile_name = "S3Profile"; - ADD_TIMER(_profile, s3_profile_name); + auto* total_time = + ADD_CHILD_TIMER(_profile, s3_profile_name, + file_scan_profile::parent_or_root(_profile, file_scan_profile::IO)); RuntimeProfile::Counter* total_get_request_counter = ADD_CHILD_COUNTER(_profile, "TotalGetRequest", TUnit::UNIT, s3_profile_name); RuntimeProfile::Counter* too_many_request_err_counter = @@ -231,6 +234,7 @@ void S3FileReader::_collect_profile_before_close() { COUNTER_UPDATE(too_many_request_sleep_time, _s3_stats.too_many_request_sleep_time_ms); COUNTER_UPDATE(total_bytes_read, _s3_stats.total_bytes_read); COUNTER_UPDATE(total_get_request_time_ns, _s3_stats.total_get_request_time_ns); + COUNTER_UPDATE(total_time, _s3_stats.total_get_request_time_ns); } } diff --git a/be/src/runtime/file_scan_profile.h b/be/src/runtime/file_scan_profile.h new file mode 100644 index 00000000000000..4ec71c4f4820d4 --- /dev/null +++ b/be/src/runtime/file_scan_profile.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "runtime/runtime_profile.h" + +namespace doris::file_scan_profile { + +inline constexpr const char* SCANNER = "FileScannerV2"; +inline constexpr const char* TABLE_READER = "TableReader"; +inline constexpr const char* FILE_READER = "FileReader"; +inline constexpr const char* IO = "IO"; + +struct Hierarchy { + RuntimeProfile::Counter* scanner = nullptr; + RuntimeProfile::Counter* table_reader = nullptr; + RuntimeProfile::Counter* file_reader = nullptr; + RuntimeProfile::Counter* io = nullptr; +}; + +inline Hierarchy ensure_hierarchy(RuntimeProfile* profile) { + if (profile == nullptr) { + return {}; + } + // RuntimeProfile stores one flat counter namespace and a separate parent map. Create every + // layer in order so later format and IO profiles cannot accidentally become scanner siblings. + auto* scanner = ADD_TIMER_WITH_LEVEL(profile, SCANNER, 1); + auto* table_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, TABLE_READER, SCANNER, 1); + auto* file_reader = ADD_CHILD_TIMER_WITH_LEVEL(profile, FILE_READER, TABLE_READER, 1); + auto* io = ADD_CHILD_TIMER_WITH_LEVEL(profile, IO, FILE_READER, 1); + return {scanner, table_reader, file_reader, io}; +} + +inline const char* parent_or_root(RuntimeProfile* profile, const char* preferred_parent) { + return profile != nullptr && profile->get_counter(preferred_parent) != nullptr + ? preferred_parent + : RuntimeProfile::ROOT_COUNTER.c_str(); +} + +} // namespace doris::file_scan_profile diff --git a/be/test/exec/scan/file_scanner_v2_test.cpp b/be/test/exec/scan/file_scanner_v2_test.cpp index 762598456e09e1..d28c3d96e74a8d 100644 --- a/be/test/exec/scan/file_scanner_v2_test.cpp +++ b/be/test/exec/scan/file_scanner_v2_test.cpp @@ -719,6 +719,15 @@ TEST(FileScannerV2Test, FileCacheStatisticsArePublishedToScannerProfile) { EXPECT_EQ(profile.get_counter("BytesWriteIntoCache")->value(), 19); ASSERT_NE(profile.get_info_string("PeerCacheNodes"), nullptr); EXPECT_EQ(*profile.get_info_string("PeerCacheNodes"), "peer-a, peer-b"); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + ASSERT_TRUE(children.contains("IO")); + EXPECT_TRUE(children.at("IO").contains("FileCache")); } TEST(FileScannerV2Test, NotFoundIsSkippedOnlyWhenConfigured) { diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 4f8f803bd3ef28..9aa793e3a04933 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -108,6 +108,13 @@ TEST(FileMetaCacheTest, KeyGenerationFromFileReader) { std::string key2 = FileMetaCache::get_key(reader2, desc2); std::string expected_key2 = FileMetaCache::get_key(file_name, 0, 300); EXPECT_EQ(key2, expected_key2); + + const std::string default_mapping = FileMetaCache::get_key(reader2, desc2, false, false); + const std::string varbinary_mapping = FileMetaCache::get_key(reader2, desc2, true, false); + const std::string timestamp_mapping = FileMetaCache::get_key(reader2, desc2, false, true); + EXPECT_NE(default_mapping, varbinary_mapping); + EXPECT_NE(default_mapping, timestamp_mapping); + EXPECT_NE(varbinary_mapping, timestamp_mapping); } TEST(FileMetaCacheTest, KeyContentVerification) { std::string file_name = "/path/to/file"; @@ -155,4 +162,4 @@ TEST(FileMetaCacheTest, InsertAndLookupWithIntValue) { EXPECT_EQ(*cached_val2, 12345); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 9d901270d5acb4..953f2a50d17755 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -26,10 +26,18 @@ #include #include "core/custom_allocator.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_struct.h" #include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" #include "format_v2/parquet/reader/native/column_reader.h" #include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/level_decoder.h" +#include "format_v2/parquet/reader/native/page_reader.h" +#include "io/fs/buffered_reader.h" #include "util/coding.h" +#include "util/thrift_util.h" namespace doris::format::parquet::native { namespace { @@ -75,6 +83,86 @@ class CaptureFixedConsumer final : public ParquetFixedValueConsumer { std::vector bytes; }; +const RowRanges& scripted_row_ranges() { + static const RowRanges ranges; + return ranges; +} + +class ScriptedColumnReader final : public ColumnReader { +public: + ScriptedColumnReader(size_t rows, size_t values, bool eof, std::vector rep_levels, + std::vector def_levels) + : ColumnReader(scripted_row_ranges(), rows, nullptr, nullptr), + _rows(rows), + _values(values), + _eof(eof), + _rep_levels(std::move(rep_levels)), + _def_levels(std::move(def_levels)) {} + + Status read_column_data(ColumnPtr& column, const DataTypePtr&, + const std::shared_ptr&, FilterMap&, + size_t, size_t* read_rows, bool* eof, bool, int64_t = -1) override { + if (_used) { + *read_rows = 0; + *eof = true; + return Status::OK(); + } + column = IColumn::mutate(std::move(column)); + column->assert_mutable()->insert_many_defaults(_values); + *read_rows = _rows; + *eof = _eof; + _used = true; + return Status::OK(); + } + + Status read_column_levels(FilterMap&, size_t, size_t* read_rows, bool* eof) override { + *read_rows = _rows; + *eof = _eof; + return Status::OK(); + } + + const std::vector& get_rep_level() const override { return _rep_levels; } + const std::vector& get_def_level() const override { return _def_levels; } + ColumnStatistics column_statistics() override { return {}; } + void close() override {} + void release_batch_scratch(size_t) override {} + void reset_filter_map_index() override {} + +private: + size_t _rows; + size_t _values; + bool _eof; + bool _used = false; + std::vector _rep_levels; + std::vector _def_levels; +}; + +class MemoryBufferedReader final : public io::BufferedStreamReader { +public: + explicit MemoryBufferedReader(std::vector data) : _data(std::move(data)) {} + + Status read_bytes(const uint8_t** buf, uint64_t offset, size_t bytes_to_read, + const io::IOContext*) override { + if (offset > _data.size() || bytes_to_read > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + *buf = _data.data() + offset; + return Status::OK(); + } + Status read_bytes(Slice& slice, uint64_t offset, const io::IOContext*) override { + if (offset > _data.size() || slice.size > _data.size() - offset) { + return Status::IOError("out of bounds"); + } + slice.data = reinterpret_cast(_data.data() + offset); + return Status::OK(); + } + std::string path() override { return "memory.parquet"; } + int64_t mtime() const override { return 0; } + +private: + std::vector _data; +}; + std::shared_ptr<::parquet::ColumnDescriptor> descriptor(::parquet::Type::type physical_type) { auto node = ::parquet::schema::PrimitiveNode::Make("value", ::parquet::Repetition::REQUIRED, physical_type); @@ -158,6 +246,12 @@ TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValida Slice empty_indices; EXPECT_TRUE(decoder.set_data(&empty_indices).is()); + + // Dictionary indices are uint32_t. Wider external bit widths would make the RLE decoder copy + // a five-byte repeated value into four-byte state before it can validate an index. + char oversized_bit_width[] = {33, 2, 0}; + Slice oversized_bit_width_slice(oversized_bit_width, sizeof(oversized_bit_width)); + EXPECT_TRUE(decoder.set_data(&oversized_bit_width_slice).is()); } TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { @@ -418,6 +512,376 @@ TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRestoresFixedWidthRows) { EXPECT_EQ(consumer.values(), std::vector({-2.5F, 3.25F})); } +TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) { + char encoded[] = {static_cast(0b00001101)}; + Slice levels(encoded, sizeof(encoded)); + LevelDecoder decoder; + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 1, 4).ok()); + + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 4), 1); + EXPECT_EQ(value, 1); + EXPECT_EQ(decoder.get_next(), 0); + decoder.rewind_one(); + EXPECT_EQ(decoder.get_next(), 0); + level_t tail[2] = {-1, -1}; + EXPECT_EQ(decoder.get_levels(tail, 2), 2); + EXPECT_EQ(tail[0], 1); + EXPECT_EQ(tail[1], 1); + + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice truncated_levels(truncated_rle, sizeof(truncated_rle)); + LevelDecoder rle_decoder; + ASSERT_TRUE(rle_decoder.init(&truncated_levels, tparquet::Encoding::RLE, 1, 1).ok()); + level_t truncated_value = -1; + EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); +} + +TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::PLAIN, decoder).ok()); + char plain_boolean[] = {static_cast(0xFF)}; + Slice plain_slice(plain_boolean, sizeof(plain_boolean)); + ASSERT_TRUE(decoder->set_data(&plain_slice).ok()); + EXPECT_FALSE(decoder->skip_values(9).ok()); + + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, decoder).ok()); + // A bit-packed run header for eight values without its required payload byte. + char truncated_rle[] = {0x01, 0x00, 0x00, 0x00, 0x03}; + Slice rle_slice(truncated_rle, sizeof(truncated_rle)); + ASSERT_TRUE(decoder->set_data(&rle_slice).ok()); + EXPECT_FALSE(decoder->skip_values(1).ok()); + + // The first literal group is complete but the second has only its header. Exact-count checks + // must reject both dense and selected decodes instead of exposing the retained vector tail. + char partially_truncated_rle[] = {0x03, 0x00, 0x00, 0x00, 0x03, static_cast(0xFF), 0x03}; + Slice partially_truncated_slice(partially_truncated_rle, sizeof(partially_truncated_rle)); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer dense_consumer; + EXPECT_FALSE(decoder->decode_fixed_values(9, dense_consumer).ok()); + ASSERT_TRUE(decoder->set_data(&partially_truncated_slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_tail { + .total_values = 9, .selected_values = 1, .ranges = {{.first = 8, .count = 1}}}; + EXPECT_FALSE(decoder->decode_selected_fixed_values(select_tail, selected_consumer).ok()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValues) { + const std::vector values {"good", "bad"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_type_length(4); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer selected_consumer; + const ParquetSelection select_good { + .total_values = 2, .selected_values = 1, .ranges = {{.first = 0, .count = 1}}}; + EXPECT_TRUE(decoder->decode_selected_fixed_values(select_good, selected_consumer) + .is()); + + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_TRUE(decoder->skip_values(2).is()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaSkipRequiresTheRequestedValueCount) { + const std::vector integers {7}; + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(integers.data(), static_cast(integers.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + Slice slice(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(2).ok()); +} + +TEST(ParquetV2NativeDecoderTest, DeltaHeadersAndLengthsAreBoundedBeforeAllocation) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(1); + // block_size=128, miniblocks=4, total_values=INT_MAX, first_value=0. + char oversized_count[] = {static_cast(0x80), + 0x01, + 0x04, + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + static_cast(0xFF), + 0x07, + 0x00}; + Slice oversized_count_slice(oversized_count, sizeof(oversized_count)); + EXPECT_TRUE(decoder->set_data(&oversized_count_slice).is()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + // A single decoded length of 1 GiB with no following payload bytes must fail before the + // decoder resizes its backing data buffer. + char oversized_length[] = {static_cast(0x80), + 0x01, + 0x04, + 0x01, + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + static_cast(0x80), + 0x08}; + Slice oversized_length_slice(oversized_length, sizeof(oversized_length)); + ASSERT_TRUE(decoder->set_data(&oversized_length_slice).ok()); + CaptureBinaryConsumer consumer; + EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_TRUE(consumer.refs.empty()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyDeltaLengthPageResetsDecoderState) { + const std::vector strings {"old", "state"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : strings) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto buffer = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, decoder) + .ok()); + Slice valid(buffer->data(), buffer->size()); + ASSERT_TRUE(decoder->set_data(&valid).ok()); + Slice empty; + EXPECT_TRUE(decoder->set_data(&empty).is()); +} + +TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRejectsPartialRowsAtPageBoundary) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + char partial_row[] = {0, 1, 2, 3, 4}; + Slice slice(partial_row, sizeof(partial_row)); + EXPECT_TRUE(decoder->set_data(&slice).is()); +} + +Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t value_rows, + size_t key_values, size_t value_values, + std::vector key_rep_levels, std::vector value_rep_levels, + bool value_eof) { + auto value_type = make_nullable(std::make_shared()); + auto map_type = std::make_shared(key_type, value_type); + ColumnPtr column = map_type->create_column(); + FieldSchema field; + field.name = "m"; + field.data_type = map_type; + field.definition_level = 1; + field.repetition_level = 1; + field.repeated_parent_def_level = 0; + + auto key_reader = std::make_unique(key_rows, key_values, true, + std::move(key_rep_levels), + std::vector(key_values, 1)); + auto value_reader = std::make_unique( + value_rows, value_values, value_eof, std::move(value_rep_levels), + std::vector(value_values, 1)); + MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); + RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); + + auto root = std::make_shared( + TableSchemaChangeHelper::ConstNode::get_instance(), + TableSchemaChangeHelper::ConstNode::get_instance()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + return reader.read_column_data(column, map_type, root, filter, key_rows, &read_rows, &eof, + false); +} + +TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingShape) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {1, 0}, {1, 0}, true) + .is()); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 0}, true) + .is()); + EXPECT_TRUE( + read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsNullKeys) { + auto nullable_key = make_nullable(std::make_shared()); + EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + FieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 0}); + children["b"] = std::make_unique(1, 1, true, std::vector {0}, + std::vector {0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_children("a", "a", TableSchemaChangeHelper::ConstNode::get_instance()); + root->add_children("b", "b", TableSchemaChangeHelper::ConstNode::get_instance()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { + constexpr size_t value_count = 1UL << 20; + std::vector encoded(value_count * sizeof(float), 0); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FLOAT, tparquet::Encoding::BYTE_STREAM_SPLIT, + decoder) + .ok()); + decoder->set_type_length(sizeof(float)); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(value_count, consumer).ok()); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + decoder->release_scratch(64UL << 10); + EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); +} + +TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption) { + auto parse_header = [](tparquet::PageHeader header) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + if (header.compressed_page_size > 0) { + bytes.resize(bytes.size() + header.compressed_page_size); + } + MemoryBufferedReader reader(bytes); + tparquet::ColumnMetaData metadata; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, bytes.size(), 1, metadata, + context); + return page_reader.parse_page_header(); + }; + + tparquet::PageHeader negative; + negative.type = tparquet::PageType::DATA_PAGE; + negative.__set_compressed_page_size(-1); + negative.__set_uncompressed_page_size(1); + negative.__isset.data_page_header = true; + negative.data_page_header.__set_num_values(1); + EXPECT_TRUE(parse_header(negative).is()); + + tparquet::PageHeader oversized_levels; + oversized_levels.type = tparquet::PageType::DATA_PAGE_V2; + oversized_levels.__set_compressed_page_size(1); + oversized_levels.__set_uncompressed_page_size(1); + oversized_levels.__isset.data_page_header_v2 = true; + oversized_levels.data_page_header_v2.__set_num_values(1); + oversized_levels.data_page_header_v2.__set_num_rows(1); + oversized_levels.data_page_header_v2.__set_repetition_levels_byte_length(2); + oversized_levels.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(oversized_levels).is()); + + tparquet::PageHeader impossible_counts = oversized_levels; + impossible_counts.__set_compressed_page_size(0); + impossible_counts.__set_uncompressed_page_size(0); + impossible_counts.data_page_header_v2.__set_repetition_levels_byte_length(0); + impossible_counts.data_page_header_v2.__set_num_nulls(2); + EXPECT_TRUE(parse_header(impossible_counts).is()); +} + +TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { + MemoryBufferedReader reader(std::vector {0}); + tparquet::ColumnMetaData metadata; + tparquet::OffsetIndex empty_index; + ParquetPageReadContext context(false, ""); + PageReader page_reader(&reader, nullptr, 0, 1, 1, metadata, context, &empty_index); + EXPECT_FALSE(page_reader.has_next_page()); + EXPECT_TRUE(page_reader.next_page().is()); +} + +TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) { + ColumnChunkReaderStatistics first_chunk; + first_chunk.page_read_counter = 1; + first_chunk.data_page_read_counter = 1; + ColumnChunkReaderStatistics second_chunk; + second_chunk.page_read_counter = 1; + second_chunk.data_page_read_counter = 1; + ColumnReader::ColumnStatistics combined; + ColumnReader::ColumnStatistics first(first_chunk, 0, 0); + ColumnReader::ColumnStatistics second(second_chunk, 0, 0); + combined.merge(first); + combined.merge(second); + + EXPECT_EQ(combined.page_read_counter, 2); + ASSERT_EQ(combined.leaf_page_read_counters.size(), 2); + EXPECT_EQ(combined.leaf_page_read_counters[0], 1); + EXPECT_EQ(combined.leaf_page_read_counters[1], 1); +} + +TEST(ParquetV2NativeDecoderTest, NativePageCacheUsesStableFileDescriptionIdentity) { + ParquetPageCacheKeyBuilder first; + ParquetPageCacheKeyBuilder replaced; + first.init("s3://bucket/object::etag-v1::1234"); + replaced.init("s3://bucket/object::etag-v2::1234"); + EXPECT_EQ(first.make_key(4096, 128).fname, "s3://bucket/object::etag-v1::1234"); + EXPECT_NE(first.make_key(4096, 128).encode(), replaced.make_key(4096, 128).encode()); +} + +TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompressed) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(100); + header.__set_uncompressed_page_size(1000); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + + // The representation is explicit in V2; a codec and compression ratio cannot override it on + // a warm cache hit. + EXPECT_TRUE(should_cache_decompressed(&header, metadata)); +} + TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchIsReleased) { ::doris::RowRanges row_ranges; tparquet::ColumnChunk chunk; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index d95f18fdcb9f55..03dbae335ce32f 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -1516,6 +1516,35 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsNullableNestedChildren) { EXPECT_TRUE(struct_type->get_element(1)->is_nullable()); } +TEST_F(NewParquetReaderTest, ComplexColumnDoesNotMisreportSiblingPagesAsCrossing) { + write_struct_filter_parquet_file(_file_path); + RuntimeProfile profile("new_parquet_reader_complex_page_fragments"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + bool eof = false; + size_t total_rows = 0; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + total_rows += rows; + } + EXPECT_EQ(total_rows, 4); + // Each STRUCT child reads one page per Row Group, but no individual leaf crosses a boundary in + // either batch. The aggregate fragment count remains useful without inflating crossing batches. + EXPECT_EQ(profile.get_counter("NativePageFragments")->value(), 8); + EXPECT_EQ(profile.get_counter("PageCrossingBatches")->value(), 0); +} + TEST_F(NewParquetReaderTest, GetSchemaMapsInt96ToTimestampTzWhenTimestampTzMappingEnabled) { write_int96_timestamp_parquet_file(_file_path); auto reader = create_reader(0, -1, nullptr, true); @@ -2019,6 +2048,45 @@ TEST_F(NewParquetReaderTest, EmptySelectionUpdatesProfileCounters) { EXPECT_EQ(profile.get_counter("EmptySelectionBatches")->value(), 1); } +TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTotalTime) { + RuntimeProfile profile("new_parquet_reader_hierarchy_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + Block block = build_file_block(schema); + auto request = std::make_shared(); + request->non_predicate_columns = {field_projection(0), field_projection(1)}; + use_schema_order_positions(request.get(), schema); + ASSERT_TRUE(reader->open(request).ok()); + + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + + auto* parquet_total = profile.get_counter("ParquetReader"); + ASSERT_NE(parquet_total, nullptr); + EXPECT_GT(parquet_total->value(), 0); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes[0].child_counters_map; + ASSERT_TRUE(children.contains(RuntimeProfile::ROOT_COUNTER)); + EXPECT_TRUE(children.at(RuntimeProfile::ROOT_COUNTER).contains("FileScannerV2")); + ASSERT_TRUE(children.contains("FileScannerV2")); + EXPECT_TRUE(children.at("FileScannerV2").contains("TableReader")); + ASSERT_TRUE(children.contains("TableReader")); + EXPECT_TRUE(children.at("TableReader").contains("FileReader")); + ASSERT_TRUE(children.contains("FileReader")); + EXPECT_TRUE(children.at("FileReader").contains("IO")); + EXPECT_TRUE(children.at("FileReader").contains("ParquetReader")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); +} + TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); diff --git a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp index c2aea1cd825253..975f1e8eddd88b 100644 --- a/be/test/io/cache/block_file_cache_profile_reporter_test.cpp +++ b/be/test/io/cache/block_file_cache_profile_reporter_test.cpp @@ -54,6 +54,16 @@ io::FileCacheStatistics make_file_cache_stats(int64_t multiplier) { stats.inverted_index_io_timer = multiplier * 28; stats.remote_only_on_miss_triggered = multiplier * 29; stats.remote_only_on_miss_threshold_bytes = multiplier * 30; + stats.num_cross_cg_peer_io_total = multiplier * 31; + stats.bytes_read_from_cross_cg_peer = multiplier * 32; + stats.cross_cg_peer_io_timer = multiplier * 33; + stats.num_same_cg_peer_io_total = multiplier * 34; + stats.bytes_read_from_same_cg_peer = multiplier * 35; + stats.same_cg_peer_io_timer = multiplier * 36; + stats.num_peer_race_peer_win = multiplier * 37; + stats.num_peer_race_s3_win = multiplier * 38; + stats.num_peer_lazy_fetch = multiplier * 39; + stats.peer_lazy_fetch_timer = multiplier * 40; return stats; } @@ -94,6 +104,16 @@ void expect_file_cache_stats_eq(const io::FileCacheStatistics& actual, EXPECT_EQ(actual.remote_only_on_miss_triggered, expected.remote_only_on_miss_triggered); EXPECT_EQ(actual.remote_only_on_miss_threshold_bytes, expected.remote_only_on_miss_threshold_bytes); + EXPECT_EQ(actual.num_cross_cg_peer_io_total, expected.num_cross_cg_peer_io_total); + EXPECT_EQ(actual.bytes_read_from_cross_cg_peer, expected.bytes_read_from_cross_cg_peer); + EXPECT_EQ(actual.cross_cg_peer_io_timer, expected.cross_cg_peer_io_timer); + EXPECT_EQ(actual.num_same_cg_peer_io_total, expected.num_same_cg_peer_io_total); + EXPECT_EQ(actual.bytes_read_from_same_cg_peer, expected.bytes_read_from_same_cg_peer); + EXPECT_EQ(actual.same_cg_peer_io_timer, expected.same_cg_peer_io_timer); + EXPECT_EQ(actual.num_peer_race_peer_win, expected.num_peer_race_peer_win); + EXPECT_EQ(actual.num_peer_race_s3_win, expected.num_peer_race_s3_win); + EXPECT_EQ(actual.num_peer_lazy_fetch, expected.num_peer_lazy_fetch); + EXPECT_EQ(actual.peer_lazy_fetch_timer, expected.peer_lazy_fetch_timer); } } // namespace @@ -139,6 +159,10 @@ TEST(FileCacheProfileReporterTest, ReporterAggregatesDeltaReportsToExactFinalTot EXPECT_EQ(profile->get_counter("CacheGetOrSetTimer")->value(), after_second_report.cache_get_or_set_timer); EXPECT_EQ(profile->get_counter("LockWaitTimer")->value(), after_second_report.lock_wait_timer); + EXPECT_EQ(profile->get_counter("CrossCGPeerIOTime")->value(), + after_second_report.cross_cg_peer_io_timer); + EXPECT_EQ(profile->get_counter("PeerLazyFetchTime")->value(), + after_second_report.peer_lazy_fetch_timer); } } // namespace doris From 42ba3ae67eb9439282cb3b79634ba82689540647 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 12:06:02 +0800 Subject: [PATCH 10/34] [fix](be) Harden and optimize native Parquet decoding ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: FileScannerV2 native Parquet scans could mis-handle malformed encoding sizes and large skips, reject valid nested MAP shapes, miss shifted STRUCT siblings, and lose legacy timestamp timezone semantics. The scan path also repeated dictionary conversion, selection bitmap construction, metadata adaptation, and predicate work, while profile counters were not always published or attributed at a diagnosable layer. Harden decoder bounds and exact-size checks, preserve complex-type and timestamp invariants, reuse persistent typed state and scratch with hysteresis, adapt predicate ordering and staged prefetch, and complete the Scanner-to-IO profile hierarchy and documentation. ### Release note Improve FileScannerV2 Parquet correctness, performance, memory stability, and scan-profile attribution. ### Check List (For Author) - Test: Unit Test and Regression test - 309 Parquet BE unit tests; 92 scanner/table-reader unit tests; 39 focused review/performance/profile unit tests; 3 related local TVF regression suites - BE and FE builds - C++ format check and diff check - Behavior changed: Yes (malformed Parquet input now fails safely; valid nested MAP/STRUCT and legacy timestamp semantics are preserved; scan scheduling and Profile attribution are optimized) - Does this need documentation: Yes (review guide and FileScannerV2 design documents are updated in this PR) --- .../data_type_number_serde.cpp | 24 +- .../data_type_serde/parquet_decode_source.h | 17 +- .../format/parquet/vparquet_file_metadata.cpp | 36 +- .../format/parquet/vparquet_file_metadata.h | 10 + .../parquet/parquet_file_context.cpp | 18 +- be/src/format_v2/parquet/parquet_profile.cpp | 2 + be/src/format_v2/parquet/parquet_reader.cpp | 25 +- be/src/format_v2/parquet/parquet_scan.cpp | 211 +++++++++--- be/src/format_v2/parquet/parquet_scan.h | 25 ++ .../format_v2/parquet/parquet_statistics.cpp | 88 ++++- be/src/format_v2/parquet/parquet_statistics.h | 2 + .../format_v2/parquet/reader/column_reader.h | 7 + .../reader/native/bool_plain_decoder.h | 3 + .../reader/native/bool_rle_decoder.cpp | 14 +- .../parquet/reader/native/bool_rle_decoder.h | 1 + .../reader/native/byte_array_dict_decoder.cpp | 8 +- .../native/byte_array_plain_decoder.cpp | 2 +- .../reader/native/byte_array_plain_decoder.h | 3 + .../reader/native/byte_stream_split_decoder.h | 3 + .../reader/native/column_chunk_reader.cpp | 25 ++ .../reader/native/column_chunk_reader.h | 17 + .../parquet/reader/native/column_reader.cpp | 102 +++++- .../parquet/reader/native/column_reader.h | 13 + .../format_v2/parquet/reader/native/decoder.h | 35 +- .../reader/native/delta_bit_pack_decoder.cpp | 84 +++-- .../reader/native/delta_bit_pack_decoder.h | 115 +++++-- .../native/fix_length_plain_decoder.cpp | 26 +- .../reader/native/fix_length_plain_decoder.h | 12 +- .../parquet/reader/native/level_decoder.h | 2 + .../parquet/reader/native_column_reader.cpp | 36 +- .../parquet/reader/native_column_reader.h | 2 + be/src/format_v2/parquet/selection_vector.h | 33 +- be/src/util/block_compression.cpp | 12 +- .../format/parquet/parquet_thrift_test.cpp | 18 + .../format_v2/parquet/native_decoder_test.cpp | 310 +++++++++++++++++- .../parquet/parquet_reader_control_test.cpp | 37 +++ .../format_v2/parquet/parquet_reader_test.cpp | 38 ++- .../format_v2/parquet/parquet_scan_test.cpp | 16 + .../parquet/parquet_statistics_test.cpp | 3 + docs/file-scanner-v2-code-review-guide.md | 37 ++- docs/file-scanner-v2-design.md | 17 + docs/file-scanner-v2-parquet-scan-design.md | 66 +++- 42 files changed, 1317 insertions(+), 238 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 8a93a5e2b624e1..398b0baff34ab2 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -204,6 +204,13 @@ Status append_parquet_number(PaddedPODArray& data, const uint8_t* size_t num_values, const ParquetDecodeContext& context) { const size_t old_size = data.size(); data.resize(old_size + num_values); + if constexpr (std::is_same_v) { + // Identical fixed-width physical/logical types need no validation or conversion. Parquet + // PLAIN values and Doris POD columns share the byte representation on supported targets, + // so preserve the dense vector-at-a-time memcpy invariant used by v1 and DuckDB. + memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); + return Status::OK(); + } for (size_t row = 0; row < num_values; ++row) { const auto value = unaligned_load(values + row * sizeof(SourceType)); if (!decoded_number_value_fits(value)) { @@ -287,6 +294,22 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { : _data(assert_cast(column).get_data()), _context(context) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + return consume_impl(values, num_values, value_width); + } + + Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) override { + // Each selected PLAIN range is already contiguous in the encoded page. Append those spans + // directly so sparse reads never build an intermediate selected-values array. + for (const auto& range : ranges) { + RETURN_IF_ERROR( + consume_impl(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } + +private: + Status consume_impl(const uint8_t* values, size_t num_values, size_t value_width) { if (_context.logical_float16) { DORIS_CHECK(_context.physical_type == ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY); DORIS_CHECK_EQ(value_width, sizeof(uint16_t)); @@ -329,7 +352,6 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { } } -private: PaddedPODArray& _data; const ParquetDecodeContext& _context; }; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index 6a0e448d1f78d0..42a95cdda7a183 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -97,12 +97,24 @@ struct ParquetDecodeContext { const cctz::time_zone* timezone = nullptr; }; +struct ParquetSelectionRange { + size_t first = 0; + size_t count = 0; +}; + // A decoder may produce multiple contiguous spans for one request (for example delta encodings). // Consumers are invoked per span, never per value, keeping virtual dispatch out of the row loop. class ParquetFixedValueConsumer { public: virtual ~ParquetFixedValueConsumer() = default; virtual Status consume(const uint8_t* values, size_t num_values, size_t value_width) = 0; + virtual Status consume_selected(const uint8_t* values, size_t value_width, + const std::vector& ranges) { + for (const auto& range : ranges) { + RETURN_IF_ERROR(consume(values + range.first * value_width, range.count, value_width)); + } + return Status::OK(); + } }; class ParquetBinaryValueConsumer { @@ -115,11 +127,6 @@ class ParquetBinaryValueConsumer { // intentionally excluded: the native ColumnReader uses this plan only when the batch has no NULL // leaf slots, so selected values can be appended in one pass without a temporary nullable column. // Ranges are sorted, disjoint, and expressed in the physical value stream's coordinate space. -struct ParquetSelectionRange { - size_t first = 0; - size_t count = 0; -}; - struct ParquetSelection { size_t total_values = 0; size_t selected_values = 0; diff --git a/be/src/format/parquet/vparquet_file_metadata.cpp b/be/src/format/parquet/vparquet_file_metadata.cpp index abca40d7f6a6e5..b204ee24186ae7 100644 --- a/be/src/format/parquet/vparquet_file_metadata.cpp +++ b/be/src/format/parquet/vparquet_file_metadata.cpp @@ -18,13 +18,17 @@ #include "format/parquet/vparquet_file_metadata.h" #include +#include +#include #include #include +#include "common/cast_set.h" #include "format/parquet/schema_desc.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" +#include "util/thrift_util.h" namespace doris { @@ -34,7 +38,7 @@ FileMetaData::FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size) } FileMetaData::~FileMetaData() { - ExecEnv::GetInstance()->parquet_meta_tracker()->release(_mem_size); + ExecEnv::GetInstance()->parquet_meta_tracker()->release(_mem_size + _arrow_metadata_mem_size); } Status FileMetaData::init_schema(const bool enable_mapping_varbinary, @@ -54,6 +58,36 @@ const tparquet::FileMetaData& FileMetaData::to_thrift() const { return _metadata; } +Status FileMetaData::get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const { + DORIS_CHECK(metadata != nullptr); + std::lock_guard lock(_arrow_metadata_mutex); + if (_arrow_metadata == nullptr) { + try { + ThriftSerializer serializer(/*compact=*/true, + static_cast(std::max(_mem_size, 4096))); + std::vector serialized_metadata; + RETURN_IF_ERROR(serializer.serialize(const_cast(&_metadata), + &serialized_metadata)); + uint32_t serialized_size = cast_set(serialized_metadata.size()); + auto parsed = + ::parquet::FileMetaData::Make(serialized_metadata.data(), &serialized_size, + ::parquet::default_reader_properties()); + DORIS_CHECK(static_cast(serialized_size) == serialized_metadata.size()); + // Native and Arrow planners describe the same immutable footer. Cache the adapter at + // the footer-cache lifecycle so repeated v2 opens do not serialize and parse it again. + _arrow_metadata_mem_size = parsed->size(); + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(_arrow_metadata_mem_size); + _arrow_metadata = std::move(parsed); + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to adapt cached Parquet metadata: {}", e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to adapt cached Parquet metadata: {}", e.what()); + } + } + *metadata = _arrow_metadata; + return Status::OK(); +} + std::string FileMetaData::debug_string() const { std::stringstream out; out << "Parquet Metadata("; diff --git a/be/src/format/parquet/vparquet_file_metadata.h b/be/src/format/parquet/vparquet_file_metadata.h index 5d0ba77aced3e9..161faa9184f503 100644 --- a/be/src/format/parquet/vparquet_file_metadata.h +++ b/be/src/format/parquet/vparquet_file_metadata.h @@ -18,11 +18,17 @@ #pragma once #include +#include +#include #include #include "common/status.h" #include "format/parquet/schema_desc.h" +namespace parquet { +class FileMetaData; +} + namespace doris { class FileMetaData { public: @@ -32,6 +38,7 @@ class FileMetaData { const FieldDescriptor& schema() const { return _schema; } FieldDescriptor& schema() { return _schema; } const tparquet::FileMetaData& to_thrift() const; + Status get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const; std::string debug_string() const; size_t get_mem_size() const { return _mem_size; } @@ -39,6 +46,9 @@ class FileMetaData { tparquet::FileMetaData _metadata; FieldDescriptor _schema; size_t _mem_size; + mutable std::mutex _arrow_metadata_mutex; + mutable std::shared_ptr<::parquet::FileMetaData> _arrow_metadata; + mutable size_t _arrow_metadata_mem_size = 0; }; } // namespace doris diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 45b9cc4c5af7cd..3957637962dd53 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -44,7 +44,6 @@ #include "runtime/exec_env.h" #include "storage/cache/page_cache.h" #include "util/slice.h" -#include "util/thrift_util.h" namespace doris::format::parquet { @@ -615,21 +614,8 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont arrow_file = std::make_shared( input_file_reader, io_ctx, enable_page_cache, std::move(page_cache_file_key)); try { - // Arrow metadata is still used by the v2 pruning planner during the migration, but it must - // not trigger a second footer read. Re-serialize the immutable cached Thrift object and - // hand the parsed Arrow metadata into Open(). The serialized buffer is needed only during - // FileMetaData::Make(), while the v1-compatible cache remains the single footer owner. - ThriftSerializer serializer(/*compact=*/true, - static_cast(std::max(native_footer_size, 4096))); - std::vector serialized_metadata; - RETURN_IF_ERROR(serializer.serialize( - const_cast(&native_metadata->to_thrift()), - &serialized_metadata)); - uint32_t serialized_size = cast_set(serialized_metadata.size()); - auto arrow_metadata = - ::parquet::FileMetaData::Make(serialized_metadata.data(), &serialized_size, - ::parquet::default_reader_properties()); - DORIS_CHECK(static_cast(serialized_size) == serialized_metadata.size()); + std::shared_ptr<::parquet::FileMetaData> arrow_metadata; + RETURN_IF_ERROR(native_metadata->get_arrow_metadata(&arrow_metadata)); this->file_reader = ::parquet::ParquetFileReader::Open( arrow_file, ::parquet::default_reader_properties(), std::move(arrow_metadata)); metadata = this->file_reader->metadata(); diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 2951fccf2eb8a2..8994c32d3d4036 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -202,12 +202,14 @@ void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_sta COUNTER_UPDATE(total_row_groups, pruning_stats.total_row_groups); COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 9bc330fa0d34f2..649d3e2b54989d 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -423,10 +423,14 @@ Status ParquetReader::init(RuntimeState* state) { } _state->scheduler.set_merge_read_options(_profile, merge_read_slice_size); _state->scheduler.set_batch_size(_batch_size); - // Open parquet file and parse metadata to get file schema. - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description, - _enable_mapping_timestamp_tz)); + // Opening the file parses the footer before any row group can be scheduled. Keep this timer + // around the whole operation so footer/cache latency cannot disappear from a slow profile. + { + SCOPED_TIMER(_parquet_profile.parse_footer_time); + RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), + _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz)); + } if (_profile != nullptr) { COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, _state->file_context.native_footer_read_calls); @@ -435,11 +439,14 @@ Status ParquetReader::init(RuntimeState* state) { } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier - RETURN_IF_ERROR( - build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); - if (_enable_mapping_timestamp_tz) { - for (auto& column_schema : _state->file_schema) { - apply_timestamp_tz_mapping(column_schema.get()); + { + SCOPED_TIMER(_parquet_profile.parse_meta_time); + RETURN_IF_ERROR( + build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); + if (_enable_mapping_timestamp_tz) { + for (auto& column_schema : _state->file_schema) { + apply_timestamp_tz_mapping(column_schema.get()); + } } } return Status::OK(); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 4c0d06eef5b0a8..199a6aab7fb076 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -38,9 +38,57 @@ #include "format_v2/parquet/reader/global_rowid_column_reader.h" #include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" +#include "util/defer_op.h" +#include "util/time.h" namespace doris::format::parquet { +namespace detail { + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats) { + if (std::ranges::any_of(positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return positions; + } + auto ordered = positions; + std::stable_sort(ordered.begin(), ordered.end(), [&](size_t left, size_t right) { + const auto score = [&](size_t position) { + const auto& sample = stats.at(position); + return sample.cost_per_input_row_ns / std::max(1.0 - sample.survival_ratio, 0.01); + }; + return score(left) < score(right); + }); + return ordered; +} + +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability) { + if (std::ranges::any_of(ordered_positions, [&](size_t position) { + const auto it = stats.find(position); + return it == stats.end() || it->second.samples == 0; + })) { + return ordered_positions; + } + std::vector result; + double reach_probability = 1; + for (const size_t position : ordered_positions) { + if (!result.empty() && reach_probability < minimum_reach_probability) { + break; + } + result.push_back(position); + reach_probability *= stats.at(position).survival_ratio; + } + return result; +} + +} // namespace detail + namespace { int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { @@ -700,6 +748,38 @@ void ParquetScanScheduler::reset_current_row_group() { _current_merge_range_active = false; } +void ParquetScanScheduler::flush_current_reader_profiles() { + for (const auto& reader : _current_predicate_columns | std::views::values) { + reader->flush_profile(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + reader->flush_profile(); + } +} + +std::vector ParquetScanScheduler::adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const { + std::vector positions; + std::unordered_map columns_by_position; + positions.reserve(request.predicate_columns.size()); + columns_by_position.reserve(request.predicate_columns.size()); + for (const auto& column : request.predicate_columns) { + const auto position_it = request.local_positions.find(column.column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + positions.push_back(position); + columns_by_position.emplace(position, &column); + } + auto ordered = detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + ordered = detail::adaptive_prefetch_prefix(ordered, _predicate_runtime_stats, 0.25); + std::vector result; + result.reserve(ordered.size()); + for (const size_t position : ordered) { + result.push_back(*columns_by_position.at(position)); + } + return result; +} + Status ParquetScanScheduler::open_next_row_group( ParquetFileContext& file_context, const std::vector>& file_schema, @@ -785,6 +865,9 @@ Status ParquetScanScheduler::open_next_row_group( for (const auto& col : request.predicate_columns) { const auto local_id = col.local_id(); + if (_current_predicate_columns.contains(local_id)) { + continue; + } if (local_id == format::ROW_POSITION_COLUMN_ID) { _current_predicate_columns[local_id] = std::make_unique( _current_row_group_first_row, _scan_profile.column_reader_profile); @@ -815,7 +898,8 @@ Status ParquetScanScheduler::open_next_row_group( // BufferedFileStreamReader later consumes the same Doris file-cache blocks; prefetch never // changes row/column materialization order. if (!_current_merge_range_active) { - prefetch_current_row_group_columns(file_context, file_schema, request.predicate_columns, + const auto prefetch_columns = adaptive_predicate_prefetch_columns(request); + prefetch_current_row_group_columns(file_context, file_schema, prefetch_columns, &_current_predicate_prefetched); } for (const auto& col : request.non_predicate_columns) { @@ -847,8 +931,9 @@ Status ParquetScanScheduler::open_next_row_group( &column_reader)); _current_non_predicate_columns[local_id] = std::move(column_reader); } - if (!_current_merge_range_active && request.conjuncts.empty() && - request.delete_conjuncts.empty()) { + if (!_current_merge_range_active && + ((request.conjuncts.empty() && request.delete_conjuncts.empty()) || + _predicate_survival_ratio >= 0.8)) { // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. @@ -1018,9 +1103,8 @@ Status filter_read_predicate_columns(Block* file_block, const std::vector(block_position), @@ -1028,14 +1112,15 @@ IColumn::Filter build_dictionary_entry_filter(size_t block_position, .data_type = column_schema.type, .values = {}}) .first->second; slot.values.reserve(1); - - for (size_t dict_idx = 0; dict_idx < fields.size(); ++dict_idx) { + for (size_t dictionary_id = 0; dictionary_id < dictionary.size(); ++dictionary_id) { + Field value; + dictionary.get(dictionary_id, value); slot.values.clear(); - slot.values.push_back(fields[dict_idx]); - dictionary_filter[dict_idx] = VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch - ? 0 - : 1; + slot.values.push_back(std::move(value)); + dictionary_filter[dictionary_id] = VExprContext::evaluate_dictionary_filter( + conjuncts, ctx) == ZoneMapFilterResult::kNoMatch + ? 0 + : 1; } return dictionary_filter; } @@ -1094,15 +1179,24 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( continue; } - ParquetDictionaryWords dict_words; + std::unique_ptr column_reader; + RETURN_IF_ERROR(NativeColumnReader::create( + *column_schema, &col, file_context.native_file, file_context.native_metadata, + row_group_idx, _current_selected_ranges, _current_offset_indexes, _timezone, + file_context.native_io_ctx, _runtime_state, file_context.native_page_cache_enabled, + file_context.native_page_cache_file_key, true, _scan_profile.column_reader_profile, + &column_reader)); + MutableColumnPtr dictionary_values; { SCOPED_TIMER(_scan_profile.dict_filter_read_dict_time); - if (!read_dictionary_words(file_context.file_reader.get(), row_group_idx, - column_schema->leaf_column_id, *column_schema, - &dict_words)) { + auto dictionary_result = column_reader->dictionary_values(); + if (!dictionary_result.has_value()) { update_counter_if_not_null(_scan_profile.dict_filter_read_failures, 1); + // Dictionary filtering is optional: a probe failure must not reject a file that + // the normal native read path can still decode. continue; } + dictionary_values = std::move(dictionary_result).value(); } // Build a safe dictionary prefilter from the dictionary-filter interface instead of @@ -1113,8 +1207,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( DictionaryResidualConjuncts residual_conjuncts; { SCOPED_TIMER(_scan_profile.dict_filter_build_time); - dictionary_filter = build_dictionary_entry_filter(block_position, *column_schema, - conjunct_it->second, dict_words); + dictionary_filter = build_dictionary_entry_filter( + block_position, *column_schema, conjunct_it->second, *dictionary_values); residual_conjuncts = build_dictionary_residual_conjuncts(conjunct_it->second); } @@ -1122,6 +1216,7 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // predicate with an integer lookup and only materialize STRING values for surviving rows. _current_dictionary_filters.emplace(local_id, std::move(dictionary_filter)); _current_dictionary_residual_conjuncts.emplace(local_id, std::move(residual_conjuncts)); + _current_predicate_columns.emplace(local_id, std::move(column_reader)); update_counter_if_not_null(_scan_profile.dict_filter_columns, 1); } return Status::OK(); @@ -1141,7 +1236,8 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, const auto schedule = build_predicate_conjunct_schedule(request); const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); - std::vector read_column_positions; + auto& read_column_positions = _read_column_positions_scratch; + read_column_positions.clear(); read_column_positions.reserve(request.predicate_columns.size()); auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, @@ -1347,7 +1443,23 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, // Single-column conjuncts can be evaluated immediately after their column is read. Once // selection shrinks, later predicate columns use ParquetColumnReader::select() so the // reader skips rows already rejected by earlier predicates instead of materializing them. + std::vector positions; + std::unordered_map indices_by_position; + positions.reserve(request.predicate_columns.size()); + indices_by_position.reserve(request.predicate_columns.size()); for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { + const auto position_it = + request.local_positions.find(request.predicate_columns[idx].column_id()); + DORIS_CHECK(position_it != request.local_positions.end()); + const size_t position = position_it->second.value(); + positions.push_back(position); + indices_by_position.emplace(position, idx); + } + const auto ordered_positions = + detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + for (size_t order_idx = 0; order_idx < ordered_positions.size(); ++order_idx) { + const size_t position = ordered_positions[order_idx]; + const size_t idx = indices_by_position.at(position); const auto& col = request.predicate_columns[idx]; const auto fid = col.local_id(); auto reader_it = _current_predicate_columns.find(fid); @@ -1355,36 +1467,45 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, auto position_it = request.local_positions.find(col.column_id()); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); + const uint16_t rows_before = *selected_rows; + const int64_t start_ns = MonotonicNanos(); bool used_dictionary_filter = false; RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, &used_dictionary_filter)); - if (*selected_rows == 0) { - for (size_t remaining_idx = idx + 1; - remaining_idx < request.predicate_columns.size(); ++remaining_idx) { - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); - auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); - DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); - RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); - } - return Status::OK(); - } const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); - if (conjunct_it == schedule.single_column_conjuncts.end()) { - continue; + if (*selected_rows != 0 && conjunct_it != schedule.single_column_conjuncts.end()) { + if (used_dictionary_filter) { + const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); + DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); + RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( + residual_it->second)); + } else { + RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + } } - if (used_dictionary_filter) { - const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); - DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); - RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( - residual_it->second)); + auto& stats = _predicate_runtime_stats[position]; + const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / + std::max(rows_before, 1); + const double survival = + static_cast(*selected_rows) / std::max(rows_before, 1); + constexpr double ADAPTIVE_ALPHA = 0.25; + if (stats.samples == 0) { + stats.cost_per_input_row_ns = cost_per_row; + stats.survival_ratio = survival; } else { - RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); + stats.cost_per_input_row_ns = ADAPTIVE_ALPHA * cost_per_row + + (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; + stats.survival_ratio = + ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; } + ++stats.samples; if (*selected_rows != 0) { continue; } - for (size_t remaining_idx = idx + 1; remaining_idx < request.predicate_columns.size(); - ++remaining_idx) { + for (size_t remaining_order_idx = order_idx + 1; + remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { + const size_t remaining_idx = + indices_by_position.at(ordered_positions[remaining_order_idx]); const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); @@ -1429,6 +1550,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows) { + // All native work performed by this batch, including early empty-selection exits, is published + // together. This preserves slow-path attribution without recursively copying statistics after + // every read/select/skip call. + Defer profile_flush {[this]() { flush_current_reader_profiles(); }}; if (_scan_profile.total_batches != nullptr) { COUNTER_UPDATE(_scan_profile.total_batches, 1); } @@ -1444,7 +1569,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( } return Status::OK(); } - SelectionVector selection; + auto& selection = _selection; DORIS_CHECK(batch_rows <= std::numeric_limits::max()); uint16_t selected_rows = static_cast(batch_rows); int64_t conjunct_filtered_rows = 0; @@ -1455,6 +1580,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( mark_condition_cache_granules(selection, selected_rows, batch_first_file_row); const bool need_filter_output = selected_rows != batch_rows; + const double batch_survival = static_cast(selected_rows) / batch_rows; + _predicate_survival_ratio = _predicate_survival_ratio < 0 + ? batch_survival + : 0.25 * batch_survival + 0.75 * _predicate_survival_ratio; if (_scan_profile.selected_rows != nullptr) { COUNTER_UPDATE(_scan_profile.selected_rows, selected_rows); } diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index f1b2d8b1c7b802..7ca51c656fe9d8 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -61,6 +61,22 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +namespace detail { +struct AdaptivePredicateStats { + double cost_per_input_row_ns = 0; + double survival_ratio = 1; + size_t samples = 0; +}; + +std::vector order_adaptive_predicates( + const std::vector& positions, + const std::unordered_map& stats); +std::vector adaptive_prefetch_prefix( + const std::vector& ordered_positions, + const std::unordered_map& stats, + double minimum_reach_probability); +} // namespace detail + // ============================================================================ // ============================================================================ @@ -153,6 +169,9 @@ class ParquetScanScheduler { private: void reset_current_row_group(); + void flush_current_reader_profiles(); + std::vector adaptive_predicate_prefetch_columns( + const format::FileScanRequest& request) const; Status open_next_row_group(ParquetFileContext& file_context, const std::vector>& file_schema, @@ -225,6 +244,12 @@ class ParquetScanScheduler { bool _enable_strict_mode = false; RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; + // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without + // reallocating selection indices, dense filter bytes, or compacted-column positions. + SelectionVector _selection; + std::vector _read_column_positions_scratch; + std::unordered_map _predicate_runtime_stats; + double _predicate_survival_ratio = -1; std::shared_ptr _condition_cache_ctx; int64_t _condition_cache_filtered_rows = 0; int64_t _predicate_filtered_rows = 0; diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index ec404f40e15ffb..ac4bf1d941a126 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -947,6 +947,58 @@ bool check_statistics(const ::parquet::RowGroupMetaData& row_group, return result == ZoneMapFilterResult::kNoMatch; } +void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::set* leaf_column_ids) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); + } + return; + } + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { + continue; + } + collect_filtered_leaf_ids(*child_schema, + format::find_child_projection(projection, child_schema->local_id), + leaf_column_ids); + } +} + +int64_t requested_compressed_bytes( + const ::parquet::RowGroupMetaData& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request) { + std::set leaf_column_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return; + } + collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); + } + + int64_t bytes = 0; + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= row_group.num_columns()) { + continue; + } + const auto column_chunk = row_group.ColumnChunk(leaf_column_id); + if (column_chunk != nullptr && column_chunk->total_compressed_size() > 0) { + bytes += column_chunk->total_compressed_size(); + } + } + return bytes; +} + Status select_row_groups_by_metadata_impl( const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, const std::vector>& file_schema, @@ -1004,6 +1056,10 @@ Status select_row_groups_by_metadata_impl( if (prune_reason != ParquetRowGroupPruneReason::NONE) { if (pruning_stats != nullptr) { pruning_stats->filtered_group_rows += row_group->num_rows(); + // FilteredBytes must describe the IO actually avoided by this scan projection; + // counting every physical child overstates savings for nested-column projection. + pruning_stats->filtered_bytes += + requested_compressed_bytes(*row_group, file_schema, request); if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { ++pruning_stats->filtered_row_groups_by_statistics; } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { @@ -1293,8 +1349,18 @@ bool select_ranges_for_expr_zonemap( return false; } const ParquetColumnSchema* column_schema = nullptr; - const auto page_indexes = - load_page_indexes_for_slot(row_group, file_schema, request, slot_index, &column_schema); + int64_t parse_page_index_time_sink = 0; + std::optional, + std::shared_ptr<::parquet::OffsetIndex>>> + page_indexes; + { + // Arrow materializes the serialized page-index objects lazily in these getters, so keep + // that cost separate from predicate evaluation when diagnosing a slow page-index scan. + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &parse_page_index_time_sink + : &pruning_stats->parse_page_index_time); + page_indexes = load_page_indexes_for_slot(row_group, file_schema, request, slot_index, + &column_schema); + } if (!page_indexes.has_value()) { return false; } @@ -1438,7 +1504,8 @@ void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexRea const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector& selected_ranges, int64_t row_group_rows, - std::map* page_skip_plans) { + std::map* page_skip_plans, + ParquetPruningStats* pruning_stats) { DORIS_CHECK(page_skip_plans != nullptr); page_skip_plans->clear(); std::vector leaf_schemas; @@ -1446,8 +1513,17 @@ void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexRea for (const auto* leaf_schema : leaf_schemas) { DORIS_CHECK(leaf_schema != nullptr); ParquetPageSkipPlan page_skip_plan; - if (build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, row_group_rows, - &page_skip_plan)) { + int64_t parse_page_index_time_sink = 0; + bool has_skip_plan = false; + { + // Offset indexes for output-only columns may not have been touched by ZoneMap + // filtering; include their lazy materialization in the same parse timer. + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &parse_page_index_time_sink + : &pruning_stats->parse_page_index_time); + has_skip_plan = build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, + row_group_rows, &page_skip_plan); + } + if (has_skip_plan) { page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); } } @@ -1540,7 +1616,7 @@ Status select_row_group_ranges_by_page_index( } if (page_skip_plans != nullptr) { build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, - row_group_rows, page_skip_plans); + row_group_rows, page_skip_plans, pruning_stats); } if (pruning_stats != nullptr) { const int64_t selected_rows = count_range_rows(*selected_ranges); diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 9e94562bf2d37b..3c918fd650abbd 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -93,6 +93,7 @@ struct ParquetPruningStats { int64_t filtered_row_groups_by_bloom_filter = 0; // row groups pruned by bloom filter int64_t filtered_row_groups_by_page_index = 0; // row groups fully pruned by page index int64_t filtered_group_rows = 0; // rows in pruned row groups + int64_t filtered_bytes = 0; // requested bytes in pruned row groups int64_t filtered_page_rows = 0; // rows pruned by page index int64_t selected_row_ranges = 0; // selected row range count int64_t page_index_read_calls = 0; // Page Index read count @@ -100,6 +101,7 @@ struct ParquetPruningStats { int64_t row_group_filter_time = 0; // row-group pruning time (ns) int64_t page_index_filter_time = 0; // page-index pruning time (ns) int64_t read_page_index_time = 0; // page-index read time (ns) + int64_t parse_page_index_time = 0; // lazy page-index materialization time (ns) int64_t expr_zonemap_unusable_evals = 0; // VExpr ZoneMap unusable evaluations int64_t in_zonemap_point_check_count = 0; // VExpr IN ZoneMap point checks int64_t in_zonemap_range_only_count = 0; // VExpr IN ZoneMap range-only checks diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 42312ce8ed8de4..ccffa2fa85e291 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -60,6 +60,13 @@ class ParquetColumnReader { MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter); + // Native statistics are cumulative and can be recursively aggregated for complex columns. + // Flush once at the scheduler batch boundary instead of snapshotting after each operation. + virtual void flush_profile() {} + virtual Result dictionary_values() { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); + } + protected: ParquetColumnReader(const ParquetColumnSchema& schema, DataTypePtr type, ParquetColumnReaderProfile profile = {}); diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h index 35d076baccf3e9..ee49a236b847ce 100644 --- a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.h @@ -65,6 +65,9 @@ class BoolPlainDecoder final : public Decoder { size_t retained_scratch_bytes() const override { return selected_values_.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { + return selected_values_.size() * sizeof(uint8_t); + } protected: inline bool _decode_value(bool* value) { diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp index 8de0ab7b9628ab..3f245d88043963 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -53,10 +53,16 @@ Status BoolRLEDecoder::set_data(Slice* slice) { } Status BoolRLEDecoder::skip_values(size_t num_values) { - _values.resize(num_values); - // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. - if (_decoder.GetBatch(_values.data(), cast_set(num_values)) != num_values) { - return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + // GetBatch reports truncation; RleDecoder::Skip assumes a valid run and can spin forever. + if (_decoder.GetBatch(_values.data(), static_cast(batch_size)) != batch_size) { + return Status::IOError("Can't skip enough booleans in Parquet RLE decoder"); + } + skipped += batch_size; } return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h index 11e2acd35383af..55d7a331bb27ca 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.h @@ -50,6 +50,7 @@ class BoolRLEDecoder final : public Decoder { release_vector_if_oversized(&_values, max_retained_bytes); } size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { return _values.size() * sizeof(uint8_t); } private: RleBatchDecoder _decoder; diff --git a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp index 94a9a50bdfbd16..cc3dc66b3cacdc 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/byte_array_dict_decoder.cpp @@ -28,14 +28,18 @@ Status ByteArrayDictDecoder::set_dict(DorisUniqueBufferPtr& dict, int32 if (_dict == nullptr) { return Status::Corruption("Wrong dictionary data for byte array type, dict is null."); } - _dict_items.reserve(num_values); if (UNLIKELY(length < 0)) { return Status::Corruption("Wrong data length in dictionary"); } const size_t dict_length = cast_set(length); + // Every BYTE_ARRAY entry needs a four-byte length prefix; bound metadata-driven reserve first. + if (UNLIKELY(num_values > dict_length / sizeof(uint32_t))) { + return Status::Corruption("Dictionary value count exceeds byte array payload"); + } + _dict_items.reserve(num_values); size_t offset_cursor = 0; char* dict_item_address = reinterpret_cast(_dict.get()); - for (int i = 0; i < num_values; ++i) { + for (size_t i = 0; i < num_values; ++i) { if (UNLIKELY(offset_cursor > dict_length || dict_length - offset_cursor < sizeof(uint32_t))) { return Status::Corruption("Wrong data length in dictionary"); diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp index 904d77ba33de9b..f3736afe892750 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -26,7 +26,7 @@ namespace doris::format::parquet::native { namespace { -Status read_length(const Slice* data, uint32_t* offset, uint32_t* length) { +Status read_length(const Slice* data, size_t* offset, uint32_t* length) { if (UNLIKELY(*offset > data->size || data->size - *offset < sizeof(uint32_t))) { return Status::IOError("Can't read byte array length from plain decoder"); } diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h index 0ad27cb2dd6a13..5bbb21a9660511 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -58,6 +58,9 @@ class ByteArrayPlainDecoder final : public Decoder { size_t retained_scratch_bytes() const override { return _binary_values.capacity() * sizeof(StringRef); } + size_t active_scratch_bytes() const override { + return _binary_values.size() * sizeof(StringRef); + } private: std::vector _binary_values; diff --git a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h index 89cf1140d46cb0..dfe1d954b61641 100644 --- a/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_stream_split_decoder.h @@ -49,6 +49,9 @@ class ByteStreamSplitDecoder final : public Decoder { size_t retained_scratch_bytes() const override { return _decoded_values.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { + return _decoded_values.size() * sizeof(uint8_t); + } private: std::vector _decoded_values; diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index a6f28248c481b5..2c5a04dd5cc447 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -412,6 +412,10 @@ Status ColumnChunkReader::load_page_data() { SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); _chunk_statistics.decompress_cnt++; RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &_page_data)); + if (UNLIKELY(_page_data.size != uncompressed_payload_size)) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_payload_size); + } } // page cache counters were incremented when PageReader did the header-only // cache lookup. Do not increment again to avoid double-counting. @@ -452,6 +456,10 @@ Status ColumnChunkReader::load_page_data() { SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); _chunk_statistics.decompress_cnt++; RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &_page_data)); + if (UNLIKELY(_page_data.size != static_cast(uncompressed_size))) { + return Status::Corruption("Parquet page decompressed to {} bytes, expected {}", + _page_data.size, uncompressed_size); + } // Decide whether to cache decompressed payload or compressed payload based on threshold bool cache_payload_decompressed = should_cache_decompressed(header, _metadata); @@ -579,6 +587,13 @@ Status ColumnChunkReader::_decode_dict_page() { // Prepare dictionary data int32_t uncompressed_size = header->uncompressed_page_size; + if (_block_compress_codec == nullptr && + UNLIKELY(header->compressed_page_size != uncompressed_size)) { + // UNCOMPRESSED pages use the compressed size as their physical copy length. + return Status::Corruption( + "Uncompressed Parquet dictionary sizes differ: compressed={}, uncompressed={}", + header->compressed_page_size, uncompressed_size); + } auto dict_data = make_unique_buffer(uncompressed_size); bool dict_loaded = false; @@ -608,6 +623,11 @@ Status ColumnChunkReader::_decode_dict_page() { // Decompress cached compressed dictionary data Slice dict_slice(dict_data.get(), uncompressed_size); RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } dict_loaded = true; } @@ -633,6 +653,11 @@ Status ColumnChunkReader::_decode_dict_page() { if (dict_num != 0) { RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &dict_slice)); + if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { + return Status::Corruption( + "Parquet dictionary decompressed to {} bytes, expected {}", + dict_slice.size, uncompressed_size); + } } // Decide whether to cache decompressed or compressed dictionary based on threshold diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index f48755050c406f..6e71ae52a6a6a4 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -163,6 +163,23 @@ class ColumnChunkReader { _def_level_decoder.release_scratch(max_retained_bytes); } + size_t retained_decoder_scratch_bytes() const { + size_t bytes = _rep_level_decoder.retained_scratch_bytes() + + _def_level_decoder.retained_scratch_bytes(); + for (const auto& [encoding, decoder] : _decoders) { + bytes += decoder->retained_scratch_bytes(); + } + return bytes; + } + + size_t active_decoder_scratch_bytes() const { + // Only the current encoding is active. Old decoder instances retain reusable capacity but + // must not make the high-water policy treat their last batch as current working memory. + return (_page_decoder == nullptr ? 0 : _page_decoder->active_scratch_bytes()) + + _rep_level_decoder.active_scratch_bytes() + + _def_level_decoder.active_scratch_bytes(); + } + tparquet::Encoding::type current_encoding() const { return _current_encoding; } ColumnChunkReaderStatistics& chunk_statistics() { diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 8b92749c0c80af..7a7b72657bcfba 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -24,12 +24,14 @@ #include #include +#include "common/cast_set.h" #include "common/status.h" #include "core/column/column.h" #include "core/column/column_array.h" #include "core/column/column_map.h" #include "core/column/column_nullable.h" #include "core/column/column_struct.h" +#include "core/column/column_vector.h" #include "core/data_type/data_type_array.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" @@ -227,10 +229,13 @@ Status init_decode_context(const FieldSchema& field, const cctz::time_zone* ctz, case tparquet::ConvertedType::TIMESTAMP_MILLIS: context->logical_type = ParquetLogicalType::TIMESTAMP; context->time_unit = ParquetTimeUnit::MILLIS; + // Legacy converted timestamps are defined as UTC-adjusted, unlike an unannotated INT64. + context->timestamp_is_adjusted_to_utc = true; break; case tparquet::ConvertedType::TIMESTAMP_MICROS: context->logical_type = ParquetLogicalType::TIMESTAMP; context->time_unit = ParquetTimeUnit::MICROS; + context->timestamp_is_adjusted_to_utc = true; break; case tparquet::ConvertedType::UINT_8: case tparquet::ConvertedType::UINT_16: @@ -263,6 +268,13 @@ Status init_decode_context(const FieldSchema& field, const cctz::time_zone* ctz, } // namespace +#ifdef BE_TEST +Status init_decode_context_for_test(const FieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context) { + return init_decode_context(field, ctz, context); +} +#endif + static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, const std::vector& rep_levels, const std::vector& def_levels) { @@ -514,6 +526,20 @@ Status ScalarColumnReader::init( template void ScalarColumnReader::release_batch_scratch( size_t max_retained_bytes) { + const size_t retained_bytes = retained_batch_scratch_bytes(); + const size_t active_bytes = active_batch_scratch_bytes(); + if (retained_bytes <= max_retained_bytes || active_bytes > max_retained_bytes) { + _oversized_scratch_idle_batches = 0; + return; + } + // An adaptive probe or one repeated outlier must not pin memory forever, but immediately + // dropping a large steady-state buffer makes every following batch allocate it again. Require + // three ordinary batches before treating an oversized capacity as idle. + constexpr uint8_t OVERSIZED_SCRATCH_IDLE_BATCHES = 3; + if (++_oversized_scratch_idle_batches < OVERSIZED_SCRATCH_IDLE_BATCHES) { + return; + } + _oversized_scratch_idle_batches = 0; if (_chunk_reader != nullptr) { // Persistent decoders also own batch-sized value/slice buffers, not only the reader. _chunk_reader->release_decoder_scratch(max_retained_bytes); @@ -542,6 +568,31 @@ void ScalarColumnReader::release_batch_scratch( } } +template +size_t ScalarColumnReader::retained_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->retained_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.capacity() * sizeof(level_t) + + _def_levels.capacity() * sizeof(level_t) + + _null_run_lengths.capacity() * sizeof(uint16_t) + + _nested_filter_map_data.capacity() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + + _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + + retained_set_bytes(_ancestor_null_indices); +} + +template +size_t ScalarColumnReader::active_batch_scratch_bytes() const { + const size_t decoder_bytes = + _chunk_reader == nullptr ? 0 : _chunk_reader->active_decoder_scratch_bytes(); + return decoder_bytes + _rep_levels.size() * sizeof(level_t) + + _def_levels.size() * sizeof(level_t) + _null_run_lengths.size() * sizeof(uint16_t) + + _nested_filter_map_data.size() * sizeof(uint8_t) + + _materialization_state.dictionary_indices.size() * sizeof(uint32_t) + + _materialization_state.selection.ranges.size() * sizeof(ParquetSelectionRange) + + _ancestor_null_indices.size() * sizeof(size_t); +} + #ifdef BE_TEST template void ScalarColumnReader::reserve_batch_scratch_for_test( @@ -558,12 +609,7 @@ void ScalarColumnReader::reserve_batch_scratch_for_ template size_t ScalarColumnReader::retained_batch_scratch_bytes_for_test() const { - return _rep_levels.capacity() * sizeof(level_t) + _def_levels.capacity() * sizeof(level_t) + - _null_run_lengths.capacity() * sizeof(uint16_t) + - _nested_filter_map_data.capacity() * sizeof(uint8_t) + - _materialization_state.dictionary_indices.capacity() * sizeof(uint32_t) + - _materialization_state.selection.ranges.capacity() * sizeof(ParquetSelectionRange) + - retained_set_bytes(_ancestor_null_indices); + return retained_batch_scratch_bytes(); } #endif @@ -898,6 +944,23 @@ ScalarColumnReader::convert_dict_column_to_string_c return result; } +template +Result ScalarColumnReader::dictionary_values() { + Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); + if (dictionary_decoder == nullptr || dictionary_decoder->dictionary_size() == 0) { + return ResultError(Status::NotSupported("Parquet column has no reusable dictionary")); + } + auto ids = ColumnInt32::create(); + auto& data = ids->get_data(); + data.resize(dictionary_decoder->dictionary_size()); + for (size_t dictionary_id = 0; dictionary_id < data.size(); ++dictionary_id) { + data[dictionary_id] = cast_set(dictionary_id); + } + // Materialize the typed dictionary once and keep it in _materialization_state. Later row-level + // filtering decodes only ids and flattens surviving values from this same dictionary. + return convert_dict_column_to_string_column(ids.get()); +} + template Status ScalarColumnReader::_try_load_dict_page(bool* loaded, bool* has_dict) { @@ -1197,11 +1260,9 @@ Status MapColumnReader::read_column_data( return Status::Corruption("Parquet map contains a null key"); } const auto& key_rep_levels = _key_reader->get_rep_level(); - const auto& value_rep_levels = _value_reader->get_rep_level(); - if (UNLIKELY(key_rep_levels != value_rep_levels)) { - return Status::Corruption("Parquet map key/value repetition shapes differ"); - } // fill offset and null map + // The key leaf is the canonical outer MAP shape. A nested value has additional repetition + // levels for its own collections, so comparing the two leaves would reject valid MAP values. RETURN_IF_ERROR(fill_array_offset(_field_schema, map.get_offsets(), null_map_ptr, key_rep_levels, _key_reader->get_def_level())); if (UNLIKELY(key_column->size() != map.get_offsets().back())) { @@ -1274,6 +1335,21 @@ Status StructColumnReader::read_column_data( size_t not_missing_orig_column_size = 0; std::vector missing_column_idxs {}; std::vector skip_reading_column_idxs {}; + std::vector reference_parent_shape; + + auto parent_shape = [this](const ColumnReader& reader) { + std::vector shape; + const auto& rep_levels = reader.get_rep_level(); + shape.reserve(rep_levels.size()); + for (const level_t rep_level : rep_levels) { + // Deeper repetitions belong to a nested child; only starts visible at this STRUCT's + // repeated-parent boundary determine how sibling values are paired. + if (rep_level <= _field_schema->repetition_level) { + shape.push_back(rep_level); + } + } + return shape; + }; _read_column_names.clear(); @@ -1314,6 +1390,7 @@ Status StructColumnReader::read_column_data( batch_size, &field_rows, &field_eof, is_dict_filter)); *read_rows = field_rows; *eof = field_eof; + reference_parent_shape = parent_shape(*_child_readers[file_name]); /* * Considering the issue in the `_read_nested_column` function where data may span across pages, leading * to missing definition and repetition levels, when filling the null_map of the struct later, it is @@ -1337,6 +1414,11 @@ Status StructColumnReader::read_column_data( return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", file_name, field_rows, *read_rows); } + if (UNLIKELY(parent_shape(*_child_readers[file_name]) != reference_parent_shape)) { + return Status::Corruption( + "Parquet struct child '{}' has a different repeated-parent shape", + file_name); + } // DCHECK_EQ(*eof, field_eof); } } diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 1a1c195bab4e83..19f78a18f1e0ed 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -49,6 +49,11 @@ namespace doris::format::parquet::native { using ::doris::FieldSchema; using ::doris::ColumnString; +#ifdef BE_TEST +Status init_decode_context_for_test(const FieldSchema& field, const cctz::time_zone* ctz, + ParquetDecodeContext* context); +#endif + class ColumnReader { public: struct ColumnStatistics { @@ -189,6 +194,9 @@ class ColumnReader { throw Exception( Status::FatalError("Method convert_dict_column_to_string_column is not supported")); } + virtual Result dictionary_values() { + return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); + } static Status create(io::FileReaderSPtr file, FieldSchema* field, const tparquet::RowGroup& row_group, const RowRanges& row_ranges, @@ -255,6 +263,7 @@ class ScalarColumnReader : public ColumnReader { bool* eof) override; Result convert_dict_column_to_string_column( const ColumnInt32* dict_column) override; + Result dictionary_values() override; const std::vector& get_rep_level() const override { return _rep_levels; } const std::vector& get_def_level() const override { return _def_levels; } ColumnStatistics column_statistics() override { @@ -362,6 +371,10 @@ class ScalarColumnReader : public ColumnReader { ColumnSelectVector _select_vector; int64_t _convert_time = 0; size_t _logical_conversion_scratch_bytes = 0; + uint8_t _oversized_scratch_idle_batches = 0; + + size_t retained_batch_scratch_bytes() const; + size_t active_batch_scratch_bytes() const; Status _skip_values(size_t num_values); Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index 3b031a68a98190..e2ec6a4d4ba9ee 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -69,6 +69,7 @@ class Decoder : public ParquetDecodeSource { virtual void release_scratch(size_t max_retained_bytes) {} virtual size_t retained_scratch_bytes() const { return 0; } + virtual size_t active_scratch_bytes() const { return 0; } virtual Status set_dict(DorisUniqueBufferPtr& dict, int32_t length, size_t num_values) { @@ -85,7 +86,8 @@ class Decoder : public ParquetDecodeSource { int32_t _type_length = -1; Slice* _data = nullptr; - uint32_t _offset = 0; + // Page offsets are host-sized so checked advances cannot wrap at the 4 GiB boundary. + size_t _offset = 0; size_t _expected_values = std::numeric_limits::max(); }; @@ -170,22 +172,31 @@ class BaseDictDecoder : public Decoder { size_t retained_scratch_bytes() const override { return _skip_indices.capacity() * sizeof(uint32_t); } + size_t active_scratch_bytes() const override { return _skip_indices.size() * sizeof(uint32_t); } protected: Status skip_values(size_t num_values) override { - _skip_indices.resize(num_values); - const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), - cast_set(num_values)); - if (UNLIKELY(skipped != num_values)) { - return Status::IOError("Can't skip enough Parquet dictionary indices"); - } + constexpr size_t kSkipBatchSize = 4096; + _skip_indices.resize(std::min(num_values, kSkipBatchSize)); const size_t num_dictionary_values = dictionary_size(); - for (size_t row = 0; row < num_values; ++row) { - if (UNLIKELY(_skip_indices[row] >= num_dictionary_values)) { - return Status::Corruption( - "Parquet dictionary index {} at skipped row {} exceeds dictionary size {}", - _skip_indices[row], row, num_dictionary_values); + size_t skipped_values = 0; + while (skipped_values < num_values) { + const size_t batch_size = std::min(num_values - skipped_values, kSkipBatchSize); + const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), + static_cast(batch_size)); + if (UNLIKELY(skipped != batch_size)) { + return Status::IOError("Can't skip enough Parquet dictionary indices"); + } + // Filter gaps may be huge RLE runs; validate them without allocating by gap size. + for (size_t row = 0; row < batch_size; ++row) { + if (UNLIKELY(_skip_indices[row] >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at skipped row {} exceeds dictionary " + "size {}", + _skip_indices[row], skipped_values + row, num_dictionary_values); + } } + skipped_values += batch_size; } return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp index 78a12593667cfd..c94aead8caa6b6 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.cpp @@ -18,21 +18,42 @@ #include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" namespace doris::format::parquet::native { -Status DeltaLengthByteArrayDecoder::_decode_lengths() { - RETURN_IF_ERROR(_len_decoder.set_bit_reader(_bit_reader)); - // get the number of encoded lengths - uint32_t num_length = _len_decoder.valid_values_count(); - _buffered_length.resize(num_length); +Status DeltaLengthByteArrayDecoder::_init_lengths() { + auto length_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_len_decoder.set_bit_reader(length_reader)); + const uint32_t num_lengths = _len_decoder.valid_values_count(); - // decode all the lengths. all the lengths are buffered in buffered_length_. - uint32_t ret; - RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), num_length, &ret)); - if (UNLIKELY(ret != num_length)) { - return Status::Corruption("Parquet delta length stream decoded {} of {} lengths", ret, - num_length); + DeltaBitPackDecoder length_locator; + length_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(length_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector lengths(std::min(num_lengths, kLocateBatchSize)); + size_t located = 0; + int64_t payload_size = 0; + while (located < num_lengths) { + const size_t batch_size = std::min(num_lengths - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR( + length_locator.decode(lengths.data(), static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + for (size_t i = 0; i < batch_size; ++i) { + if (UNLIKELY(lengths[i] < 0) || + common::add_overflow(payload_size, static_cast(lengths[i]), + payload_size)) { + return Status::Corruption("Invalid Parquet delta byte-array length"); + } + } + located += batch_size; + } + // The locator leaves the shared reader at the first payload byte without retaining every + // length; the independent decoder supplies bounded batches during materialization or skip. + if (UNLIKELY(payload_size > _bit_reader->bytes_left())) { + return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", + payload_size, _bit_reader->bytes_left()); } - _length_idx = 0; - _num_valid_values = num_length; + _num_valid_values = num_lengths; return Status::OK(); } @@ -47,7 +68,14 @@ Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, } int64_t data_size = 0; - const int32_t* length_ptr = _buffered_length.data() + _length_idx; + _buffered_length.resize(max_values); + uint32_t lengths_decoded = 0; + RETURN_IF_ERROR(_len_decoder.decode(_buffered_length.data(), static_cast(max_values), + &lengths_decoded)); + if (UNLIKELY(lengths_decoded != max_values)) { + return Status::Corruption("Parquet delta length stream ended early"); + } + const int32_t* length_ptr = _buffered_length.data(); for (int i = 0; i < max_values; ++i) { int32_t len = length_ptr[i]; if (len < 0) [[unlikely]] { @@ -64,8 +92,6 @@ Status DeltaLengthByteArrayDecoder::_get_internal(Slice* buffer, int max_values, return Status::Corruption("Parquet delta lengths require {} bytes, only {} remain", data_size, _bit_reader->bytes_left()); } - _length_idx += max_values; - _buffered_data.resize(data_size); char* data_ptr = _buffered_data.data(); for (int64_t j = 0; j < data_size; j++) { @@ -101,16 +127,32 @@ Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* } int64_t data_size = 0; - const int32_t* prefix_len_ptr = _buffered_prefix_length.data() + _prefix_len_offset; + _buffered_prefix_length.resize(max_values); + uint32_t prefixes_decoded = 0; + RETURN_IF_ERROR(_prefix_len_decoder.decode( + _buffered_prefix_length.data(), static_cast(max_values), &prefixes_decoded)); + if (UNLIKELY(prefixes_decoded != max_values)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + const int32_t* prefix_len_ptr = _buffered_prefix_length.data(); + size_t preceding_value_size = _last_value.size(); for (int i = 0; i < max_values; ++i) { if (prefix_len_ptr[i] < 0) [[unlikely]] { return Status::InvalidArgument("negative prefix length in DELTA_BYTE_ARRAY"); } - if (common::add_overflow(data_size, static_cast(prefix_len_ptr[i]), data_size) || - common::add_overflow(data_size, static_cast(buffer[i].size), data_size)) + const size_t prefix_size = static_cast(prefix_len_ptr[i]); + if (prefix_size > preceding_value_size) [[unlikely]] { + // Prefixes form a dependency chain, so validate each one before aggregate allocation. + return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); + } + size_t reconstructed_size = 0; + if (common::add_overflow(prefix_size, buffer[i].size, reconstructed_size) || + reconstructed_size > static_cast(std::numeric_limits::max()) || + common::add_overflow(data_size, static_cast(reconstructed_size), data_size)) [[unlikely]] { return Status::InvalidArgument("excess expansion in DELTA_BYTE_ARRAY"); } + preceding_value_size = reconstructed_size; } _buffered_data.resize(data_size); @@ -118,9 +160,6 @@ Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* char* data_ptr = _buffered_data.data(); for (int i = 0; i < max_values; ++i) { - if (static_cast(prefix_len_ptr[i]) > prefix.length()) [[unlikely]] { - return Status::InvalidArgument("prefix length too large in DELTA_BYTE_ARRAY"); - } memcpy(data_ptr, prefix.data(), prefix_len_ptr[i]); // buffer[i] currently points to the string suffix memcpy(data_ptr + prefix_len_ptr[i], buffer[i].data, buffer[i].size); @@ -129,7 +168,6 @@ Status DeltaByteArrayDecoder::_get_internal(Slice* buffer, int max_values, int* data_ptr += buffer[i].size; prefix = std::string_view {buffer[i].data, buffer[i].size}; } - _prefix_len_offset += max_values; _num_valid_values -= max_values; _last_value = std::string {prefix}; diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index 68c8ebd7e5a44c..dbc5fef2536cf2 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -66,14 +66,20 @@ class DeltaBitPackDecoder final : public DeltaDecoder { ~DeltaBitPackDecoder() override = default; Status skip_values(size_t num_values) override { - _values.resize(num_values); - uint32_t num_valid_values = 0; - RETURN_IF_ERROR( - _get_internal(_values.data(), cast_set(num_values), &num_valid_values)); - // Skips have the same exact-consumption contract as materialized decodes. - if (UNLIKELY(num_valid_values != num_values)) { - return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", - num_values, num_valid_values); + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + uint32_t num_valid_values = 0; + RETURN_IF_ERROR(_get_internal(_values.data(), static_cast(batch_size), + &num_valid_values)); + // Skips retain exact validation without allocating in proportion to a sparse gap. + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError("Expected to skip {} Parquet delta values, skipped {}", + num_values, skipped + num_valid_values); + } + skipped += batch_size; } return Status::OK(); } @@ -127,6 +133,9 @@ class DeltaBitPackDecoder final : public DeltaDecoder { size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(T) + _delta_bit_widths.size() * sizeof(uint8_t); + } Status set_data(Slice* slice) override { _bit_reader.reset( @@ -187,13 +196,20 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { } Status skip_values(size_t num_values) override { - _values.resize(num_values); - int num_valid_values = 0; - RETURN_IF_ERROR( - _get_internal(_values.data(), cast_set(num_values), &num_valid_values)); - if (UNLIKELY(num_valid_values != num_values)) { - return Status::IOError("Expected to skip {} Parquet delta-length values, skipped {}", - num_values, num_valid_values); + constexpr size_t kSkipBatchSize = 4096; + _values.resize(std::min(num_values, kSkipBatchSize)); + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + int num_valid_values = 0; + RETURN_IF_ERROR( + _get_internal(_values.data(), static_cast(batch_size), &num_valid_values)); + if (UNLIKELY(num_valid_values != batch_size)) { + return Status::IOError( + "Expected to skip {} Parquet delta-length values, skipped {}", num_values, + skipped + num_valid_values); + } + skipped += batch_size; } return Status::OK(); } @@ -240,6 +256,8 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { return _get_internal(buffer, num_values, out_num_values); } + int valid_values_count() const { return _num_valid_values; } + void release_scratch(size_t max_retained_bytes) override { release_vector_if_oversized(&_values, max_retained_bytes); release_vector_if_oversized(&_string_refs, max_retained_bytes); @@ -252,6 +270,11 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { _buffered_length.capacity() * sizeof(int32_t) + _buffered_data.capacity() * sizeof(char) + _len_decoder.retained_scratch_bytes(); } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _buffered_length.size() * sizeof(int32_t) + _buffered_data.size() * sizeof(char) + + _len_decoder.active_scratch_bytes(); + } Status set_data(Slice* slice) override { if (slice == nullptr || slice->size == 0) { @@ -260,7 +283,6 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { _data = nullptr; _offset = 0; _num_valid_values = 0; - _length_idx = 0; _buffered_length.clear(); _buffered_data.clear(); return Status::Corruption("Parquet delta-length page is empty"); @@ -268,20 +290,18 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); _data = slice; _offset = 0; - RETURN_IF_ERROR(_decode_lengths()); + RETURN_IF_ERROR(_init_lengths()); return Status::OK(); } Status set_bit_reader(std::shared_ptr bit_reader) { _bit_reader = std::move(bit_reader); - RETURN_IF_ERROR(_decode_lengths()); + RETURN_IF_ERROR(_init_lengths()); return Status::OK(); } private: - // Decode all the encoded lengths. The decoder_ will be at the start of the encoded data - // after that. - Status _decode_lengths(); + Status _init_lengths(); Status _get_internal(Slice* buffer, int max_values, int* out_num_values); std::vector _values; @@ -290,7 +310,6 @@ class DeltaLengthByteArrayDecoder final : public DeltaDecoder { DeltaBitPackDecoder _len_decoder; int _num_valid_values; - uint32_t _length_idx; std::vector _buffered_length; std::vector _buffered_data; }; @@ -306,8 +325,15 @@ class DeltaByteArrayDecoder : public DeltaDecoder { } Status skip_values(size_t num_values) override { - RETURN_IF_ERROR(_decode_slices(num_values)); - return _validate_fixed_width_values(); + constexpr size_t kSkipBatchSize = 4096; + size_t skipped = 0; + while (skipped < num_values) { + const size_t batch_size = std::min(num_values - skipped, kSkipBatchSize); + RETURN_IF_ERROR(_decode_slices(batch_size)); + RETURN_IF_ERROR(_validate_fixed_width_values()); + skipped += batch_size; + } + return Status::OK(); } Status decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) override { @@ -373,22 +399,36 @@ class DeltaByteArrayDecoder : public DeltaDecoder { Status set_data(Slice* slice) override { _bit_reader = std::make_shared((const uint8_t*)slice->data, slice->size); - RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(_bit_reader)); + auto prefix_reader = std::make_shared(*_bit_reader); + RETURN_IF_ERROR(_prefix_len_decoder.set_bit_reader(prefix_reader)); // get the number of encoded prefix lengths int num_prefix = _prefix_len_decoder.valid_values_count(); - // call _prefix_len_decoder.Decode to decode all the prefix lengths. - // all the prefix lengths are buffered in _buffered_prefix_length. - _buffered_prefix_length.resize(num_prefix); - uint32_t ret; - RETURN_IF_ERROR( - _prefix_len_decoder.decode(_buffered_prefix_length.data(), num_prefix, &ret)); - DCHECK_EQ(ret, num_prefix); - _prefix_len_offset = 0; + DeltaBitPackDecoder prefix_locator; + prefix_locator.set_expected_values(_expected_values); + RETURN_IF_ERROR(prefix_locator.set_bit_reader(_bit_reader)); + constexpr size_t kLocateBatchSize = 4096; + std::vector ignored_prefixes( + std::min(static_cast(num_prefix), kLocateBatchSize)); + size_t located = 0; + while (located < static_cast(num_prefix)) { + const size_t batch_size = + std::min(static_cast(num_prefix) - located, kLocateBatchSize); + uint32_t decoded = 0; + RETURN_IF_ERROR(prefix_locator.decode(ignored_prefixes.data(), + static_cast(batch_size), &decoded)); + if (UNLIKELY(decoded != batch_size)) { + return Status::Corruption("Parquet delta prefix stream ended early"); + } + located += batch_size; + } _num_valid_values = num_prefix; // at this time, the decoder_ will be at the start of the encoded suffix data. RETURN_IF_ERROR(_suffix_decoder.set_bit_reader(_bit_reader)); + if (UNLIKELY(_suffix_decoder.valid_values_count() != num_prefix)) { + return Status::Corruption("Parquet delta prefix and suffix counts differ"); + } // TODO: read corrupted files written with bug(PARQUET-246). _last_value should be set // to _last_value_in_previous_page when decoding a new page(except the first page) @@ -422,6 +462,14 @@ class DeltaByteArrayDecoder : public DeltaDecoder { _prefix_len_decoder.retained_scratch_bytes() + _suffix_decoder.retained_scratch_bytes(); } + size_t active_scratch_bytes() const override { + return _values.size() * sizeof(Slice) + _string_refs.size() * sizeof(StringRef) + + _fixed_values.size() * sizeof(uint8_t) + + _buffered_prefix_length.size() * sizeof(int32_t) + + _buffered_data.size() * sizeof(char) + _last_value.size() + + _last_value_in_previous_page.size() + _prefix_len_decoder.active_scratch_bytes() + + _suffix_decoder.active_scratch_bytes(); + } private: Status _validate_fixed_width_values() const { @@ -462,7 +510,6 @@ class DeltaByteArrayDecoder : public DeltaDecoder { // string buffer for last value in previous page std::string _last_value_in_previous_page; int _num_valid_values; - uint32_t _prefix_len_offset; std::vector _buffered_prefix_length; std::vector _buffered_data; }; diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp index 09f019c7dc0033..831b6268b6310c 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -17,7 +17,6 @@ #include "format_v2/parquet/reader/native/fix_length_plain_decoder.h" -#include #include namespace doris::format::parquet::native { @@ -46,28 +45,21 @@ Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelectio if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { return Status::IOError("Out-of-bounds access in Parquet plain selection decoder"); } - _selected_values.resize(selection.selected_values * value_width); - size_t output_offset = 0; - for (const auto& range : selection.ranges) { - const size_t range_bytes = range.count * value_width; - memcpy(_selected_values.data() + output_offset, - reinterpret_cast(_data->data) + _offset + range.first * value_width, - range_bytes); - output_offset += range_bytes; - } - DORIS_CHECK_EQ(output_offset, _selected_values.size()); + const auto* values = reinterpret_cast(_data->data) + _offset; _offset += input_bytes; - if (_selected_values.empty()) { - return Status::OK(); - } - return consumer.consume(_selected_values.data(), selection.selected_values, value_width); + // PLAIN pages are random-access fixed-width spans. Keep those page bytes pinned while the + // consumer gathers directly into the final column, otherwise sparse scans pay for a second + // selected-width buffer and copy before materialization. + return consumer.consume_selected(values, value_width, selection.ranges); } Status FixLengthPlainDecoder::skip_values(size_t num_values) { - _offset += _type_length * num_values; - if (UNLIKELY(_offset > _data->size)) { + DORIS_CHECK(_type_length > 0); + const size_t value_width = static_cast(_type_length); + if (UNLIKELY(_offset > _data->size || num_values > (_data->size - _offset) / value_width)) { return Status::IOError("Out-of-bounds access in parquet data decoder"); } + _offset += num_values * value_width; return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h index cd59e22785f9df..fbe7d12c3522e2 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -19,8 +19,6 @@ #include -#include - #include "common/status.h" #include "core/column/column_fixed_length_object.h" #include "core/data_type/data_type.h" @@ -45,15 +43,7 @@ class FixLengthPlainDecoder final : public Decoder { Status skip_values(size_t num_values) override; - void release_scratch(size_t max_retained_bytes) override { - release_vector_if_oversized(&_selected_values, max_retained_bytes); - } - size_t retained_scratch_bytes() const override { - return _selected_values.capacity() * sizeof(uint8_t); - } - -private: - std::vector _selected_values; + size_t retained_scratch_bytes() const override { return 0; } }; } // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h index 69cbe02e9f9ddd..ee389a07c13a54 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.h +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -55,6 +55,8 @@ class LevelDecoder { std::vector().swap(_rle_scratch); } } + size_t retained_scratch_bytes() const { return _rle_scratch.capacity() * sizeof(uint16_t); } + size_t active_scratch_bytes() const { return _rle_scratch.size() * sizeof(uint16_t); } private: tparquet::Encoding::type _encoding; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index f100cd780c1f38..68335ee014d861 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -350,8 +350,6 @@ Status NativeColumnReader::read(int64_t rows, MutableColumnPtr& column, int64_t* RETURN_IF_ERROR(read_with_filter(rows, nullptr, false, column, _type, false, rows_read)); advance_selected_span(*rows_read); update_reader_read_rows(*rows_read); - int64_t max_leaf_page_reads = 0; - record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } @@ -395,23 +393,18 @@ Status NativeColumnReader::skip(int64_t rows) { remaining -= rows_read; } update_reader_skip_rows(native_skipped_rows); - int64_t max_leaf_page_reads = 0; - record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } Status NativeColumnReader::select(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, MutableColumnPtr& column) { - RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); RETURN_IF_ERROR(validate_selected_span(batch_rows)); - _filter_scratch.assign(static_cast(batch_rows), 0); - for (uint16_t idx = 0; idx < selected_rows; ++idx) { - _filter_scratch[selection.get_index(idx)] = 1; - } + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); const size_t old_size = column->size(); int64_t rows_read = 0; - RETURN_IF_ERROR(read_with_filter(batch_rows, _filter_scratch.data(), selected_rows == 0, column, - _type, false, &rows_read)); + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, column, _type, + false, &rows_read)); advance_selected_span(rows_read); if (column->size() != old_size + selected_rows) { return Status::Corruption( @@ -423,8 +416,6 @@ Status NativeColumnReader::select(const SelectionVector& selection, uint16_t sel } update_reader_read_rows(selected_rows); update_reader_skip_rows(batch_rows - selected_rows); - int64_t max_leaf_page_reads = 0; - record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); return Status::OK(); } @@ -436,7 +427,6 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& bool* used_filter) { DORIS_CHECK(row_filter != nullptr); DORIS_CHECK(used_filter != nullptr); - RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); RETURN_IF_ERROR(validate_selected_span(batch_rows)); *used_filter = false; row_filter->clear(); @@ -445,10 +435,8 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& } *used_filter = true; - _filter_scratch.assign(static_cast(batch_rows), 0); - for (uint16_t idx = 0; idx < selected_rows; ++idx) { - _filter_scratch[selection.get_index(idx)] = 1; - } + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); const bool nullable = _type->is_nullable(); DataTypePtr id_type = std::make_shared(); if (nullable) { @@ -459,7 +447,7 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& } _dictionary_id_column->clear(); int64_t rows_read = 0; - RETURN_IF_ERROR(read_with_filter(batch_rows, _filter_scratch.data(), selected_rows == 0, + RETURN_IF_ERROR(read_with_filter(batch_rows, filter_data, selected_rows == 0, _dictionary_id_column, id_type, true, &rows_read)); advance_selected_span(rows_read); if (_dictionary_id_column->size() != selected_rows) { @@ -514,9 +502,17 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& } update_reader_read_rows(cast_set(matched_ids.size())); update_reader_skip_rows(batch_rows - cast_set(matched_ids.size())); + return Status::OK(); +} + +void NativeColumnReader::flush_profile() { int64_t max_leaf_page_reads = 0; record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); - return Status::OK(); +} + +Result NativeColumnReader::dictionary_values() { + DORIS_CHECK(_native_reader != nullptr); + return _native_reader->dictionary_values(); } void NativeColumnReader::record_page_fragments(int64_t page_fragments, diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 3fe6e6e91ff338..f561620719a67f 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -78,6 +78,8 @@ class NativeColumnReader final : public ParquetColumnReader { const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter) override; + void flush_profile() override; + Result dictionary_values() override; private: NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, diff --git a/be/src/format_v2/parquet/selection_vector.h b/be/src/format_v2/parquet/selection_vector.h index b77a1198d22ccb..ab2bf93c785edc 100644 --- a/be/src/format_v2/parquet/selection_vector.h +++ b/be/src/format_v2/parquet/selection_vector.h @@ -17,6 +17,7 @@ #include #include +#include #include #include @@ -66,6 +67,7 @@ class SelectionVector { _owned.clear(); _data = data; _size = count; + ++_generation; } void resize(size_t count) { @@ -75,12 +77,14 @@ class SelectionVector { for (size_t idx = 0; idx < count; ++idx) { _data[idx] = static_cast(idx); } + ++_generation; } void clear() { _owned.clear(); _data = nullptr; _size = 0; + ++_generation; } size_t size() const { return _size; } @@ -98,7 +102,29 @@ class SelectionVector { return _data[idx]; } - void set_index(size_t idx, Index value) { _data[idx] = value; } + void set_index(size_t idx, Index value) { + _data[idx] = value; + ++_generation; + } + + Status materialize_filter(size_t count, int64_t batch_rows, const uint8_t** filter) const { + DORIS_CHECK(filter != nullptr); + RETURN_IF_ERROR(verify(count, batch_rows)); + if (_filter_generation != _generation || _filter_count != count || + _filter_batch_rows != batch_rows) { + // Selection is shared by all readers in one scheduler batch. Cache its dense bitmap so + // a wide lazy projection does not rebuild the same O(batch_rows) filter per column. + _filter.assign(static_cast(batch_rows), 0); + for (size_t idx = 0; idx < count; ++idx) { + _filter[get_index(idx)] = 1; + } + _filter_generation = _generation; + _filter_count = count; + _filter_batch_rows = batch_rows; + } + *filter = _filter.data(); + return Status::OK(); + } Status verify(size_t count, int64_t batch_rows) const { if (batch_rows < 0) { @@ -135,6 +161,11 @@ class SelectionVector { std::vector _owned; Index* _data = nullptr; size_t _size = 0; + uint64_t _generation = 0; + mutable std::vector _filter; + mutable uint64_t _filter_generation = std::numeric_limits::max(); + mutable size_t _filter_count = 0; + mutable int64_t _filter_batch_rows = -1; }; inline void selection_to_ranges(const SelectionVector& selection, uint16_t selected_rows, diff --git a/be/src/util/block_compression.cpp b/be/src/util/block_compression.cpp index abb64cff9e6a71..47c67a011a1701 100644 --- a/be/src/util/block_compression.cpp +++ b/be/src/util/block_compression.cpp @@ -769,11 +769,19 @@ class SnappyBlockCompression : public BlockCompressionCodec { } Status decompress(const Slice& input, Slice* output) override { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(input.data, input.size, &uncompressed_size)) { + return Status::InvalidArgument("Fail to get Snappy uncompressed length"); + } + // RawUncompress has no capacity argument, so reject an undersized destination first. + if (uncompressed_size > output->size) { + return Status::InvalidArgument("Snappy output size {} exceeds buffer capacity {}", + uncompressed_size, output->size); + } if (!snappy::RawUncompress(input.data, input.size, output->data)) { return Status::InvalidArgument("Fail to do Snappy decompress"); } - // NOTE: GetUncompressedLength only takes O(1) time - snappy::GetUncompressedLength(input.data, input.size, &output->size); + output->size = uncompressed_size; return Status::OK(); } diff --git a/be/test/format/parquet/parquet_thrift_test.cpp b/be/test/format/parquet/parquet_thrift_test.cpp index 0fe101db598138..d9609648fe432a 100644 --- a/be/test/format/parquet/parquet_thrift_test.cpp +++ b/be/test/format/parquet/parquet_thrift_test.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -96,6 +97,23 @@ TEST_F(ParquetThriftReaderTest, normal) { } } +TEST_F(ParquetThriftReaderTest, ReusesArrowMetadataAdapterForCachedNativeFooter) { + auto local_fs = io::global_local_filesystem(); + io::FileReaderSPtr reader; + ASSERT_TRUE(local_fs->open_file("./be/test/exec/test_data/parquet_scanner/localfile.parquet", + &reader) + .ok()); + std::unique_ptr metadata; + size_t meta_size = 0; + ASSERT_TRUE(parse_thrift_footer(reader, &metadata, &meta_size, nullptr, true, true).ok()); + + std::shared_ptr<::parquet::FileMetaData> first; + std::shared_ptr<::parquet::FileMetaData> second; + ASSERT_TRUE(metadata->get_arrow_metadata(&first).ok()); + ASSERT_TRUE(metadata->get_arrow_metadata(&second).ok()); + EXPECT_EQ(first, second); +} + TEST_F(ParquetThriftReaderTest, complex_nested_file) { // hive-complex.parquet is the part of following table: // complex_nested_table( diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 953f2a50d17755..55bed1de3e81d3 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -22,10 +22,12 @@ #include #include +#include #include #include #include "core/custom_allocator.h" +#include "core/data_type/data_type_date_or_datetime_v2.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" @@ -36,8 +38,11 @@ #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" #include "io/fs/buffered_reader.h" +#include "util/block_compression.h" #include "util/coding.h" +#include "util/faststring.h" #include "util/thrift_util.h" +#include "util/timezone_utils.h" namespace doris::format::parquet::native { namespace { @@ -203,6 +208,37 @@ std::vector encode_plain_byte_arrays(const std::vector& va return encoded; } +Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, + tparquet::CompressionCodec::type codec) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + const size_t chunk_size = bytes.size(); + MemoryBufferedReader reader(std::move(bytes)); + + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(codec); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(chunk_size); + if (header.type == tparquet::PageType::DICTIONARY_PAGE) { + chunk.meta_data.__set_dictionary_page_offset(0); + chunk.meta_data.__set_data_page_offset(0); + } else { + chunk.meta_data.__set_data_page_offset(0); + } + FieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + return chunk_reader.load_page_data(); +} + TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { int32_t dictionary_length = 0; auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); @@ -254,6 +290,46 @@ TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValida EXPECT_TRUE(decoder.set_data(&oversized_bit_width_slice).is()); } +TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryBoundsEntryCountBeforeReserve) { + auto dictionary = make_unique_buffer(1); + dictionary.get()[0] = 0; + ByteArrayDictDecoder decoder; + EXPECT_TRUE(decoder.set_dict(dictionary, 1, std::numeric_limits::max()) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { + TimezoneUtils::load_timezones_to_cache(); + for (const auto converted_type : + {tparquet::ConvertedType::TIMESTAMP_MILLIS, tparquet::ConvertedType::TIMESTAMP_MICROS}) { + FieldSchema field; + field.physical_type = tparquet::Type::INT64; + field.parquet_schema.__set_converted_type(converted_type); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.logical_type, ParquetLogicalType::TIMESTAMP); + EXPECT_TRUE(context.timestamp_is_adjusted_to_utc); + + cctz::time_zone shanghai; + ASSERT_TRUE(TimezoneUtils::find_cctz_time_zone("Asia/Shanghai", shanghai)); + ASSERT_TRUE(init_decode_context_for_test(field, &shanghai, &context).ok()); + int64_t epoch = 0; + Slice epoch_slice(reinterpret_cast(&epoch), sizeof(epoch)); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(sizeof(epoch)); + ASSERT_TRUE(decoder->set_data(&epoch_slice).ok()); + ParquetMaterializationState state; + DataTypeDateTimeV2 type(0); + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, *decoder, context, 1, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 08:00:00"); + } +} + TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { const ParquetSelection selection { .total_values = 7, @@ -318,6 +394,45 @@ TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCurso EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); } +TEST(ParquetV2NativeDecoderTest, SparsePlainFixedDecodeDoesNotRetainGatherBuffer) { + constexpr size_t value_count = 1UL << 18; + std::vector integers(value_count); + std::iota(integers.begin(), integers.end(), 0); + Slice integer_slice(reinterpret_cast(integers.data()), + integers.size() * sizeof(int32_t)); + + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_data(&integer_slice).ok()); + const ParquetSelection selection { + .total_values = value_count, + .selected_values = 2, + .ranges = {{.first = 1, .count = 1}, {.first = value_count - 1, .count = 1}}}; + CaptureFixedConsumer selected_integers; + ASSERT_TRUE(decoder->decode_selected_fixed_values(selection, selected_integers).ok()); + EXPECT_EQ(selected_integers.values(), + std::vector({1, static_cast(value_count - 1)})); + // Sparse PLAIN spans can be consumed directly from the encoded page. Retaining a gather buffer + // makes a highly selective batch allocate in proportion to its selected width for no benefit. + EXPECT_EQ(decoder->retained_scratch_bytes(), 0); +} + +TEST(ParquetV2NativeDecoderTest, FixedPlainLargeSkipCannotWrapPageOffset) { + std::vector values {11, 22}; + Slice slice(reinterpret_cast(values.data()), values.size() * sizeof(int64_t)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT64, tparquet::Encoding::PLAIN, decoder).ok()); + decoder->set_type_length(sizeof(int64_t)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + EXPECT_FALSE(decoder->skip_values(536870913).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + EXPECT_EQ(consumer.values(), std::vector({11})); +} + TEST(ParquetV2NativeDecoderTest, PlainAndBooleanRleExposeRawValuesAndPreserveCursor) { std::unique_ptr decoder; ASSERT_TRUE( @@ -568,6 +683,86 @@ TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { EXPECT_FALSE(decoder->decode_selected_fixed_values(select_tail, selected_consumer).ok()); } +TEST(ParquetV2NativeDecoderTest, CompactRleSkipsKeepScratchBounded) { + constexpr uint32_t value_count = 1U << 20; + uint8_t header[16]; + uint8_t* header_end = encode_varint32(header, value_count << 1); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({"only"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 1).ok()); + std::vector dictionary_indices {0}; + dictionary_indices.insert(dictionary_indices.end(), reinterpret_cast(header), + reinterpret_cast(header_end)); + Slice dictionary_slice(dictionary_indices.data(), dictionary_indices.size()); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + ParquetDecodeSource& dictionary_source = dictionary_decoder; + ASSERT_TRUE(dictionary_source.skip_values(value_count).ok()); + EXPECT_LE(dictionary_decoder.retained_scratch_bytes(), 4096 * sizeof(uint32_t)); + + std::vector boolean_payload(reinterpret_cast(header), + reinterpret_cast(header_end)); + boolean_payload.push_back(0); + std::vector boolean_page(sizeof(uint32_t) + boolean_payload.size()); + encode_fixed32_le(reinterpret_cast(boolean_page.data()), boolean_payload.size()); + memcpy(boolean_page.data() + sizeof(uint32_t), boolean_payload.data(), boolean_payload.size()); + Slice boolean_slice(boolean_page.data(), boolean_page.size()); + std::unique_ptr boolean_decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::BOOLEAN, tparquet::Encoding::RLE, boolean_decoder) + .ok()); + ASSERT_TRUE(boolean_decoder->set_data(&boolean_slice).ok()); + ASSERT_TRUE(boolean_decoder->skip_values(value_count).ok()); + EXPECT_LE(boolean_decoder->retained_scratch_bytes(), 4096); +} + +TEST(ParquetV2NativeDecoderTest, CompactDeltaSkipKeepsScratchBounded) { + constexpr uint32_t delta_count = 1U << 20; + std::vector encoded(64); + uint8_t* cursor = encoded.data(); + cursor = encode_varint32(cursor, delta_count); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, delta_count + 1); + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta + *cursor++ = 0; // zero-width miniblock + encoded.resize(cursor - encoded.data()); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + decoder->set_expected_values(delta_count + 1); + Slice slice(encoded.data(), encoded.size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(delta_count + 1).ok()); + EXPECT_LE(decoder->retained_scratch_bytes(), 4096 * sizeof(int32_t) + 1); +} + +TEST(ParquetV2NativeDecoderTest, DeltaByteArraySkipsKeepScratchBounded) { + constexpr size_t value_count = 1U << 16; + std::vector<::parquet::ByteArray> values(value_count); + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + for (const auto encoding : + {::parquet::Encoding::DELTA_LENGTH_BYTE_ARRAY, ::parquet::Encoding::DELTA_BYTE_ARRAY}) { + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>(encoding, false, + byte_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + static_cast(encoding), decoder) + .ok()); + decoder->set_expected_values(value_count); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->skip_values(value_count).ok()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); + } +} + TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValues) { const std::vector values {"good", "bad"}; std::vector<::parquet::ByteArray> byte_arrays; @@ -650,10 +845,29 @@ TEST(ParquetV2NativeDecoderTest, DeltaHeadersAndLengthsAreBoundedBeforeAllocatio static_cast(0x80), 0x08}; Slice oversized_length_slice(oversized_length, sizeof(oversized_length)); - ASSERT_TRUE(decoder->set_data(&oversized_length_slice).ok()); CaptureBinaryConsumer consumer; - EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_TRUE(decoder->set_data(&oversized_length_slice).is()); EXPECT_TRUE(consumer.refs.empty()); + + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(1); + std::vector huge_prefix(64); + uint8_t* cursor = huge_prefix.data(); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, std::numeric_limits::max() - 1); + cursor = encode_varint32(cursor, 128); + cursor = encode_varint32(cursor, 4); + cursor = encode_varint32(cursor, 1); + cursor = encode_varint32(cursor, 0); // one empty suffix + huge_prefix.resize(cursor - huge_prefix.data()); + Slice huge_prefix_slice(huge_prefix.data(), huge_prefix.size()); + ASSERT_TRUE(decoder->set_data(&huge_prefix_slice).ok()); + EXPECT_TRUE(decoder->decode_binary_values(1, consumer).is()); + EXPECT_LT(decoder->retained_scratch_bytes(), 1UL << 20); } TEST(ParquetV2NativeDecoderTest, EmptyDeltaLengthPageResetsDecoderState) { @@ -723,12 +937,13 @@ Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t va false); } -TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingShape) { +TEST(ParquetV2NativeDecoderTest, MapReaderUsesKeyShapeForNestedValues) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 0}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { auto int_type = std::make_shared(); - EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {1, 0}, {1, 0}, true) - .is()); - EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 0}, true) - .is()); EXPECT_TRUE( read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); } @@ -769,6 +984,36 @@ TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { .is()); } +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + FieldSchema field; + field.name = "s"; + field.repetition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique( + 2, 3, true, std::vector {0, 1, 0}, std::vector {0, 0, 0}); + children["b"] = std::make_unique( + 2, 3, true, std::vector {0, 0, 1}, std::vector {0, 0, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_children("a", "a", TableSchemaChangeHelper::ConstNode::get_instance()); + root->add_children("b", "b", TableSchemaChangeHelper::ConstNode::get_instance()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { constexpr size_t value_count = 1UL << 20; std::vector encoded(value_count * sizeof(float), 0); @@ -782,6 +1027,16 @@ TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { CaptureFixedConsumer consumer; ASSERT_TRUE(decoder->decode_fixed_values(value_count, consumer).ok()); ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); + EXPECT_EQ(decoder->active_scratch_bytes(), value_count * sizeof(float)); + + std::vector ordinary_encoded(sizeof(float), 0); + Slice ordinary_slice(ordinary_encoded.data(), ordinary_encoded.size()); + ASSERT_TRUE(decoder->set_data(&ordinary_slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + // Capacity records the high-water allocation while size records this batch. The reader needs + // both to distinguish stable large batches from a one-off outlier before releasing capacity. + EXPECT_EQ(decoder->active_scratch_bytes(), sizeof(float)); + ASSERT_GT(decoder->retained_scratch_bytes(), 1UL << 20); decoder->release_scratch(64UL << 10); EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); } @@ -829,6 +1084,41 @@ TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption EXPECT_TRUE(parse_header(impossible_counts).is()); } +TEST(ParquetV2NativeDecoderTest, PageDecompressionRejectsBothSizeMismatchDirections) { + BlockCompressionCodec* codec = nullptr; + ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); + std::vector input(8, 7); + faststring compressed; + ASSERT_TRUE(codec->compress(Slice(input.data(), input.size()), &compressed).ok()); + std::vector payload(compressed.data(), compressed.data() + compressed.size()); + + for (const int32_t advertised_size : {4, 12}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(advertised_size); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + EXPECT_FALSE(load_scripted_page(header, payload, tparquet::CompressionCodec::SNAPPY).ok()); + } +} + +TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndLogicalSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DICTIONARY_PAGE; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(1); + header.__isset.dictionary_page_header = true; + header.dictionary_page_header.__set_num_values(1); + header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + EXPECT_TRUE(load_scripted_page(header, std::vector(8), + tparquet::CompressionCodec::UNCOMPRESSED) + .is()); +} + TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { MemoryBufferedReader reader(std::vector {0}); tparquet::ColumnMetaData metadata; @@ -882,7 +1172,7 @@ TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompres EXPECT_TRUE(should_cache_decompressed(&header, metadata)); } -TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchIsReleased) { +TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { ::doris::RowRanges row_ranges; tparquet::ColumnChunk chunk; ScalarColumnReader reader(row_ranges, 1, chunk, nullptr, nullptr, nullptr); @@ -892,6 +1182,10 @@ TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchIsReleased) { const size_t oversized_bytes = reader.retained_batch_scratch_bytes_for_test(); ASSERT_GT(oversized_bytes, max_retained_bytes); + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); + reader.release_batch_scratch(max_retained_bytes); + EXPECT_EQ(reader.retained_batch_scratch_bytes_for_test(), oversized_bytes); reader.release_batch_scratch(max_retained_bytes); const size_t released_bytes = reader.retained_batch_scratch_bytes_for_test(); EXPECT_LT(released_bytes, oversized_bytes); diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 72ab5e58034eea..2d187785bed723 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -27,6 +27,7 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type_number.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" @@ -68,14 +69,18 @@ class CursorColumnReader final : public ParquetColumnReader { return Status::OK(); } + void flush_profile() override { ++_profile_flushes; } + int64_t cursor() const { return _cursor; } const std::vector& skip_lengths() const { return _skip_lengths; } const std::vector& read_lengths() const { return _read_lengths; } + int profile_flushes() const { return _profile_flushes; } private: int64_t _cursor = 0; std::vector _skip_lengths; std::vector _read_lengths; + int _profile_flushes = 0; }; GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { @@ -140,6 +145,28 @@ TEST(SelectionVectorTest, VerifyRejectsInvalidSelection) { EXPECT_FALSE(selection.verify(2, 3).ok()); } +TEST(SelectionVectorTest, MaterializedFilterIsReusedUntilSelectionChanges) { + SelectionVector selection(4); + selection.set_index(0, 1); + selection.set_index(1, 3); + const uint8_t* first_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &first_filter).ok()); + ASSERT_NE(first_filter, nullptr); + EXPECT_EQ(std::vector(first_filter, first_filter + 4), + std::vector({0, 1, 0, 1})); + + const uint8_t* reused_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &reused_filter).ok()); + EXPECT_EQ(reused_filter, first_filter); + + selection.set_index(1, 2); + const uint8_t* updated_filter = nullptr; + ASSERT_TRUE(selection.materialize_filter(2, 4, &updated_filter).ok()); + EXPECT_EQ(updated_filter, first_filter); + EXPECT_EQ(std::vector(updated_filter, updated_filter + 4), + std::vector({0, 1, 1, 0})); +} + TEST(ParquetColumnReaderControlTest, BaseSelectUsesSkipReadRanges) { CursorColumnReader reader; SelectionVector selection(3); @@ -171,6 +198,16 @@ TEST(ParquetColumnReaderControlTest, BaseSelectZeroRowsConsumesBatch) { EXPECT_EQ(reader.skip_lengths(), std::vector({4})); } +TEST(ParquetColumnReaderControlTest, SchedulerFlushesReaderProfilesAtBatchBoundary) { + ParquetScanScheduler scheduler; + auto reader = std::make_unique(); + auto* reader_ptr = reader.get(); + scheduler._current_predicate_columns.emplace(0, std::move(reader)); + + scheduler.flush_current_reader_profiles(); + EXPECT_EQ(reader_ptr->profile_flushes(), 1); +} + TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { RowPositionColumnReader reader(100); EXPECT_EQ(reader.file_column_id(), format::ROW_POSITION_COLUMN_ID); diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 03dbae335ce32f..d68e8522e0ad15 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -1438,6 +1438,36 @@ TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchC EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); } +TEST_F(NewParquetReaderTest, NativeNestedMapUsesOuterKeyRepetitionShape) { + const char* source_root = std::getenv("ROOT"); + ASSERT_NE(source_root, nullptr); + _file_path = + std::string(source_root) + "/regression-test/data/external_table_p0/tvf/comp.parquet"; + ASSERT_TRUE(std::filesystem::exists(_file_path)); + + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + for (size_t position = 0; position < schema.size(); ++position) { + request->non_predicate_columns.push_back(field_projection(cast_set(position))); + } + ASSERT_TRUE(reader->open(request).ok()); + + size_t total_rows = 0; + bool eof = false; + while (!eof) { + Block block = build_file_block(schema); + size_t rows = 0; + const Status status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + total_rows += rows; + } + EXPECT_GT(total_rows, 0); +} + TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { write_decimal_and_fixed_binary_parquet_file(_file_path); RuntimeProfile profile("native_decimal_fixed_binary_materialization"); @@ -2437,7 +2467,7 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { EXPECT_GE(profile.get_counter("ReaderSelectRows")->value(), 8); } -TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) { +TEST_F(NewParquetReaderTest, DictionaryPredicateReaderIsSharedOutsideMergeRangeReader) { write_dictionary_filter_with_trailing_column_parquet_file(_file_path); RuntimeProfile profile("new_parquet_reader_dictionary_filter_merge_profile"); @@ -2479,9 +2509,9 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateProbeDoesNotUseMergeRangeReader) EXPECT_EQ(values, std::vector({"az", "za"})); EXPECT_EQ(payloads, std::vector({20, 50})); EXPECT_EQ(profile.get_counter("RowsFilteredByDictFilter")->value(), 4); - // Dictionary probing finishes on the base Arrow reader before the native data-page path - // installs its row-group-scoped merge reader. The merge profile therefore describes only - // native projected chunk reads and cannot be perturbed by dictionary ReadAt order. + // The native dictionary probe keeps its reader and cursor for predicate data pages. Other + // projected chunks still use the row-group merge reader, so probing never duplicates the + // dictionary read or perturbs the merge range's sequential access order. ASSERT_NE(profile.get_counter("MergedIO"), nullptr); ASSERT_NE(profile.get_counter("MergedBytes"), nullptr); EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 5532fe57dd2b5e..7c3007eb732b2f 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -522,6 +522,22 @@ TEST(ParquetScanSelectionTest, CompactFilterShrinksCurrentSelection) { EXPECT_TRUE(selection.verify(selected_rows, 6).ok()); } +TEST(ParquetScanAdaptivePredicateTest, OrdersByObservedCostPerRejectedRow) { + using format::parquet::detail::AdaptivePredicateStats; + std::unordered_map stats; + stats.emplace(0, AdaptivePredicateStats { + .cost_per_input_row_ns = 10, .survival_ratio = 0.8, .samples = 3}); + stats.emplace(1, AdaptivePredicateStats { + .cost_per_input_row_ns = 20, .survival_ratio = 0.1, .samples = 3}); + stats.emplace(2, AdaptivePredicateStats { + .cost_per_input_row_ns = 50, .survival_ratio = 0.5, .samples = 3}); + + const auto order = format::parquet::detail::order_adaptive_predicates({0, 1, 2}, stats); + EXPECT_EQ(order, std::vector({1, 0, 2})); + const auto prefetched = format::parquet::detail::adaptive_prefetch_prefix(order, stats, 0.25); + EXPECT_EQ(prefetched, std::vector({1})); +} + TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { write_int_pair_parquet_file(_file_path, 2); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index dd2138279b11d1..4bb9399045ea0c 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -321,6 +321,8 @@ class BloomInExpr final : public VExpr { format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { format::FileScanRequest request; + request.predicate_columns.push_back( + format::LocalColumnIndex::top_level(format::LocalColumnId(0))); request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); return request; @@ -770,6 +772,7 @@ TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRo .ok()); EXPECT_EQ(selected, std::vector({2})); EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); + EXPECT_GT(pruning_stats.filtered_bytes, 0); selected.clear(); ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index 169469d7d033ca..b3d50f2c4154f8 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -99,10 +99,19 @@ format-specific checklist when reviewing Parquet or ORC. - Keep index construction, predicate translation, cache lookup, and virtual-column setup out of per-row and repeated batch paths unless the work is inherently row-local. Avoid repeated schema traversal, expression cloning, metadata parsing, allocation, and conversion. +- Keep Profile counters in the visible `FileScannerV2 -> TableReader -> FileReader -> IO` hierarchy. + Format-specific readers, such as `ParquetReader`, belong below `FileReader`; lifecycle, metadata, + index, predicate, decode, materialization, and physical I/O paths must all have timers at the layer + that owns the work. Flush recursively aggregated child-reader statistics at every batch boundary, + including empty-selection and error exits, so a slow in-progress scan is diagnosable before close. - Require format readers to populate the common `ReaderStatistics` accurately where applicable: filtered/read row groups, Bloom and min/max pruning, filtered group/page/lazy rows, read rows and bytes, metadata/footer/cache timing, page-index work, predicate time, dictionary rewrite, and Bloom read time. +- Reject dead or ambiguous counters. In particular, `FilteredBytes` counts compressed bytes of + projected physical chunks avoided by pruning, not every child in the Row Group; footer read, + footer parse, lazy page-index materialization, and page-index predicate evaluation need distinct + timers. Raw I/O counters stay under `IO` even when a format reader initiates the request. - Evaluate performance with representative format versions, writers, data ordering, predicate selectivity, nested width, remote storage, batch sizes, and warm/cold caches. Report both the optimization overhead and the avoided work; a low pruning ratio alone is not a defect. @@ -117,6 +126,9 @@ format-specific checklist when reviewing Parquet or ORC. - Verify physical/logical metadata is immutable per leaf reader and complete for signed integers, decimal precision/scale, date/time/timestamp units and UTC adjustment, INT96, UUID, FLOAT16, and fixed-width binary. Unsupported combinations return explicit errors before plausible output. +- Treat legacy Parquet `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` converted types as UTC-adjusted. + Do not give them the local/unspecified semantics of an unannotated INT64 timestamp; data decode, + statistics conversion, and min/max pruning must use the same timezone rule. - Verify schema-change routing separately from physical decode. Integer, FLOAT-to-DOUBLE, decimal, and string-family changes should use the direct target-SerDe path. Other supported logical casts may use one persistent generic `ColumnTypeConverter` source column; its value/null-map sizes must @@ -125,6 +137,8 @@ format-specific checklist when reviewing Parquet or ORC. - Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate every index before access, and invalidate cached dictionary state at Row Group/file/type changes. + Dictionary-entry predicate evaluation and later row-value flattening must reuse that same typed + generation rather than serializing, parsing, or converting the dictionary twice. - Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded values without allocating output; null runs must append defaults without advancing payload. @@ -139,9 +153,23 @@ format-specific checklist when reviewing Parquet or ORC. - Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct without materializing an intermediate complex column. +- For MAP, the non-null key leaf owns the outer entry shape. Validate the materialized key/value + entry counts, but do not compare their raw repetition vectors: a nested value legitimately has + deeper levels. For STRUCT, compare each sibling only at the current parent boundary and ignore + repetition owned by a deeper child collection. - Require a bounded high-water policy for persistent definition/repetition, null, selection, - conversion, and dictionary-index scratch. Test that an oversized repeated-value batch releases - retained capacity without discarding ordinary reusable capacity. + conversion, dictionary-index, and decoder-owned scratch. Distinguish active bytes from retained + capacity: never release an oversized buffer while the current batch still needs it, and require + three ordinary/idle batches before releasing capacity above the high-water limit. Test both + outlier release and steady-state reuse so the policy does not create allocation thrash. +- Review all decoder read and skip paths as equally exposed corruption boundaries. Check requested + counts against remaining/declared values before pointer arithmetic or narrowing; use checked + addition/multiplication for byte extents; bound BYTE_ARRAY dictionary entry counts and IDs before + allocation/indexing; and require DELTA_BYTE_ARRAY prefixes to fit the previous reconstructed + value. BOOLEAN RLE and DELTA skip paths must consume bounded chunks and fail on short streams. +- Before Snappy decompression, inspect the encoded uncompressed length and validate destination + capacity. Page V1, Page V2, and dictionary pages must produce exactly the declared decoded size; + an UNCOMPRESSED dictionary page must also declare equal compressed and uncompressed sizes. - For a STRUCT whose projected children are all missing after schema evolution, require a levels-only physical reference leaf. It must advance and validate encoded payload cursors while deriving the synthetic child count, without constructing a discarded string/complex column. @@ -172,6 +200,11 @@ format-specific checklist when reviewing Parquet or ORC. - Verify lazy materialization avoids reading and decoding non-predicate columns for rejected rows while advancing all readers correctly. Predicate columns should be read/prefetched first; output prefetch should wait for survivors when filtering is active. +- For safe staged single-column predicates, review the observed cost/rejection ordering and its + cold-start behavior. Reordering is allowed only after every candidate has a sample; prefetch may + stop at a low-probability reach prefix, while output-column prefetch may start early only after a + learned high survival ratio. Cache the batch SelectionVector's dense bitmap by generation so a + wide lazy projection does not rebuild the same O(batch) filter for every column. - Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. diff --git a/docs/file-scanner-v2-design.md b/docs/file-scanner-v2-design.md index 66b96de1204d14..b2fe43ae76f6b0 100644 --- a/docs/file-scanner-v2-design.md +++ b/docs/file-scanner-v2-design.md @@ -282,6 +282,23 @@ Scan optimization remains maintainable only when costs are visible, sources are and failure semantics are explicit. V2 provides three complementary views: Query Profile, query resource context, and global metrics. +Query Profile uses one stable ownership tree: + +```text +FileScannerV2 +└── TableReader + └── FileReader + ├── format-specific reader (ParquetReader, OrcReader, ...) + └── IO +``` + +Scanner lifecycle and Split scheduling are charged to `FileScannerV2`; table-schema restoration, +delete handling, and reader lifecycle are charged to `TableReader`; metadata, index, decode, and +materialization are charged to `FileReader` and its format subtree; physical reads, bytes, cache +waits, and remote/local attribution remain under `IO`. Every executable lifecycle path needs a +timer, and cumulative child-reader statistics must be published at batch boundaries as well as +close, so an active slow query never presents an unexplained timing gap. + ```mermaid flowchart LR R[FileReader and FileCache Raw Statistics] --> P[Query Profile] diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 5ba0ca9ad621cd..c67ebeb1578adf 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -158,7 +158,9 @@ sequenceDiagram - **ParquetFileContext:** Adapts Doris FileReader to the active metadata/data stream interface and owns Page Cache, FileCache prefetch, and MergeRange routing. During migration this may still expose an Arrow adapter for metadata consumers, but native page decoding reads the same stable - Doris byte ranges without producing Arrow arrays. + Doris byte ranges without producing Arrow arrays. The immutable native footer owns one + thread-safe, lazily constructed Arrow metadata adapter at the footer-cache lifecycle, so repeated + v2 opens neither re-read the footer nor serialize and parse the same metadata again. For every selected Row Group, dictionary/index probes finish on the metadata adapter first. The scheduler then computes projected physical Column Chunk ranges and installs one shared native @@ -411,6 +413,20 @@ Arrow and never produce a plausible result through a decoder selected only by lo Dictionary-to-plain transitions, multiple data pages, Page V1/V2, truncated payloads, integer overflow, and invalid lengths/IDs are part of the unit-test matrix. +Read and skip are the same trust boundary. Every decoder validates the requested count against its +declared/remaining values before pointer advancement, allocation, multiplication, or integer +narrowing. BYTE_ARRAY dictionaries bound the entry count by the available four-byte length prefixes +before reserving storage and validate every decoded ID. DELTA_BYTE_ARRAY requires each prefix to fit +the preceding reconstructed value and checks the reconstructed and aggregate byte lengths. BOOLEAN +RLE and DELTA skip paths operate in bounded chunks and reject short streams instead of advancing a +partially consumed cursor as if the request succeeded. + +Compression has an exact-size contract. Snappy's encoded uncompressed length is checked against the +destination capacity before decompression; Page V1, Page V2, and dictionary decode must then produce +exactly the size declared in the page header. For the UNCOMPRESSED codec, dictionary compressed and +uncompressed sizes must be equal. These rules turn malformed size metadata into corruption instead +of a buffer overrun, silent truncation, or plausible shifted values. + ### 7.3 Direct Materialization and Scratch Reuse Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected @@ -437,15 +453,25 @@ Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, two's-complement values with correct sign extension, and apply precision/scale conversion exactly once. Date, timestamp, INT96, unsigned annotations, CHAR/VARCHAR, and timezone conversions retain the same semantic checks as the general conversion path. A fast path is enabled only when those -checks prove the result is equivalent. +checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and +`TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct +local/unspecified semantics; data decode and statistics pruning share this interpretation. + +The typed dictionary is materialized through the logical SerDe once per decoder dictionary +generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving +row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation +change invalidates it, and every ID is checked before access. The persistent leaf reader owns reusable conversion objects, null map, selection ranges, definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris column capacity. Logical sizes are reset at batch boundaries and normal-size capacity is retained. -After the top-level complex reader has consumed the level plan, any individual batch scratch buffer -above the 4 MiB high-water limit is released; this prevents staggered repeated-value outliers from -accumulating one retained allocation per leaf for the rest of the Row Group. The native path does -not create an Arrow builder or Arrow array. +After the top-level complex reader has consumed the level plan, retained capacity above the 4 MiB +high-water limit becomes eligible for release. The reader accounts separately for active and +retained bytes, including decoder-owned buffers: active oversized scratch is never released, and +eligible capacity is released only after three consecutive ordinary/idle batches. This hysteresis +prevents a repeated-value outlier from being pinned for the Row Group without turning a legitimate +large steady-state batch into allocate/free thrash. The native path does not create an Arrow +builder or Arrow array. ### 7.4 Complex Types and Parent Shape Plans @@ -463,8 +489,11 @@ ARRAY and MAP offsets and null maps are derived from that plan. Parquet stores s streams for separate physical leaves, so sibling readers still consume their own streams; they must advance over the same parent-row range and validate their payload counts instead of redefining the parent shape. STRUCT uses a representative present leaf for its null map and parent count. MAP uses -the key leaf as the entry-shape owner, requires the value leaf to produce the same entry count, and -validates Parquet's non-null key requirement rather than repairing it. +the key leaf as the entry-shape owner, requires the materialized key and value columns to match that +outer entry count, and validates Parquet's non-null key requirement rather than repairing it. Raw +key/value repetition vectors are intentionally not compared because a nested MAP value owns +additional repetition levels. STRUCT siblings similarly normalize repetition to the current parent +boundary; deeper child collection repetition cannot redefine or invalidate the sibling shape. Level scratch is sized by decoded level entries, not by the 16-bit parent batch cap. Long repeated rows and long null/non-null runs are split into representable internal runs without introducing a @@ -706,6 +735,18 @@ Without row-level filtering, output columns may be warmed together. With filteri columns first and defer non-predicate columns until at least one row survives, aligning network bandwidth with lazy materialization. +For safe single-column conjuncts, the scheduler records an exponentially weighted cost per input +row and survival ratio. Cold batches preserve declaration order; after every candidate has a sample, +the next batch orders predicates by cost per rejected row. The same learned order limits predicate +prefetch to the prefix with a meaningful probability of being reached, and a sustained high overall +survival ratio estimate may warm output chunks early. This retains correctness because only +independently safe conjuncts move and all readers still advance over the same logical batch. + +Selection state is scheduler-owned across adaptive batches. Its dense filter bitmap is cached by +selection generation and batch shape, so every lazily materialized output reader consumes the same +bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary +capacity remains reusable. + ## 11. Correctness, Fallback, and Capability Boundaries V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be @@ -740,6 +781,13 @@ split safely, or read anomalies must never change query semantics. Troubleshoot in this order: verify planning effectiveness, row filtering, lazy materialization, and then I/O/cache health. Total ScanTime alone does not identify the cause. +The visible timer hierarchy is `FileScannerV2 -> TableReader -> FileReader -> IO`; the +format-specific `ParquetReader` subtree belongs to `FileReader`. Scanner lifecycle and Split work, +table-semantic restoration, format metadata/index/decode/materialization, and physical I/O are +charged to their owning layer. Recursive native child-reader statistics are flushed at every batch +boundary, including empty-selection and early-return paths, so a query cannot be slow merely because +its counters are waiting for reader close. + ```mermaid flowchart TD A[Slow Scan] --> B{Many Row Groups Pruned?} @@ -763,6 +811,8 @@ flowchart TD | Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | +| Metadata lifecycle | How much time was spent reading the footer, parsing metadata, lazily materializing page indexes, and evaluating page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | From a1de9bdc97856b881a84fbeda3f41e4729b746e3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 14:09:17 +0800 Subject: [PATCH 11/34] [fix](be) Address Parquet v2 review feedback --- be/src/core/column/column_string.h | 30 ++++ .../data_type_datetimev2_serde.cpp | 63 +++++-- .../data_type_datev2_serde.cpp | 37 +++- .../data_type_decimal_serde.cpp | 39 ++++- .../data_type_number_serde.cpp | 95 ++++++++--- .../data_type_string_serde.cpp | 32 ++-- .../data_type_serde/data_type_time_serde.cpp | 39 ++++- .../data_type_serde/parquet_decode_source.h | 33 ++++ be/src/format/parquet/parquet_thrift_util.h | 12 +- .../format/parquet/vparquet_file_metadata.cpp | 33 ++-- .../format/parquet/vparquet_file_metadata.h | 7 +- .../parquet/parquet_file_context.cpp | 26 ++- .../parquet/reader/count_column_reader.cpp | 4 +- .../reader/native/column_chunk_reader.cpp | 140 +++++++++++++-- .../reader/native/column_chunk_reader.h | 22 ++- .../parquet/reader/native/column_reader.cpp | 58 ++++--- .../parquet/reader/native/column_reader.h | 6 +- .../parquet/reader/native/level_reader.cpp | 32 ++-- .../parquet/reader/native/level_reader.h | 2 +- .../parquet/reader/native/page_reader.cpp | 4 +- .../parquet/reader/native/page_reader.h | 28 ++- .../parquet/reader/native_column_reader.cpp | 6 +- .../data_type_serde_parquet_test.cpp | 91 ++++++++++ .../format_v2/parquet/native_decoder_test.cpp | 159 ++++++++++++++++++ docs/file-scanner-v2-code-review-guide.md | 23 +++ docs/file-scanner-v2-parquet-scan-design.md | 27 ++- 26 files changed, 899 insertions(+), 149 deletions(-) diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index 9ca55caa9c6aa5..b864a20debfe18 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -322,6 +323,35 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + void insert_many_fixed_length_data(const char* data, size_t value_length, size_t num) { + if (num == 0) { + return; + } + if (value_length > std::numeric_limits::max() / num) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + const size_t old_chars_size = chars.size(); + const size_t old_offsets_size = offsets.size(); + const size_t bytes = value_length * num; + if (bytes > std::numeric_limits::max() - old_chars_size || + num > std::numeric_limits::max() - old_offsets_size) { + throw doris::Exception(ErrorCode::INTERNAL_ERROR, + "ColumnString fixed-length append size overflow"); + } + check_chars_length(old_chars_size + bytes, old_offsets_size + num); + chars.resize(old_chars_size + bytes); + if (bytes != 0) { + memcpy(chars.data() + old_chars_size, data, bytes); + } + offsets.resize(old_offsets_size + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_offsets_size + row] = + static_cast(old_chars_size + (row + 1) * value_length); + } + sanity_check_simple(); + } + template void insert_many_strings_fixed_length(const StringRef* strings, size_t num) { size_t new_size = 0; diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index ad22905c4acb0a..f5f1e83909d941 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -22,6 +22,7 @@ #include // IWYU pragma: keep #include +#include #include "common/config.h" #include "common/status.h" @@ -135,20 +136,31 @@ int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { return value; } -int64_t parquet_timestamp_micros(const ParquetDecodeContext& context, int64_t value) { +Status parquet_timestamp_micros(const ParquetDecodeContext& context, int64_t value, + int64_t* result) { if (context.time_unit == ParquetTimeUnit::MILLIS) { - return value * 1000; + if (value > std::numeric_limits::max() / 1000 || + value < std::numeric_limits::min() / 1000) { + return Status::DataQualityError("Parquet timestamp overflows microseconds"); + } + *result = value * 1000; + return Status::OK(); } if (context.time_unit == ParquetTimeUnit::NANOS) { - return value / 1000; + *result = value / 1000; + return Status::OK(); } - return value; + *result = value; + return Status::OK(); } class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { public: - DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context) - : _data(assert_cast(column).get_data()), _context(context) {} + DateTimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { const size_t old_size = _data.size(); @@ -166,14 +178,27 @@ class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { } DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); DORIS_CHECK_EQ(value_width, sizeof(int64_t)); - timestamp_micros = parquet_timestamp_micros( - _context, unaligned_load(values + row * sizeof(int64_t))); + auto status = parquet_timestamp_micros( + _context, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } if (_context.timestamp_is_adjusted_to_utc) { append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, timezone); continue; } - auto status = append_datetimev2_from_epoch_micros(_data, timestamp_micros); + status = append_datetimev2_from_epoch_micros(_data, timestamp_micros); if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } _data.resize(old_size); return status; } @@ -184,6 +209,7 @@ class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { private: ColumnDateTimeV2::Container& _data; const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; }; class RejectDateTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { @@ -676,20 +702,35 @@ Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, context.physical_type != ParquetPhysicalType::INT96) { return Status::NotSupported("DATETIMEV2 expects Parquet INT64 or INT96"); } - DateTimeV2ParquetConsumer consumer(column, context); + DateTimeV2ParquetConsumer consumer(column, context, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { return source.decode_fixed_values(num_values, consumer); } if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); - RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateTimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectDateTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < num_values; ++row) { + if (!state.dictionary_conversion_failures.empty() && + state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + state.mark_conversion_failure(old_size + row); + } + } + } return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp index de975078477394..c7e0478c156053 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp @@ -45,8 +45,8 @@ namespace { class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { public: - explicit DateV2ParquetConsumer(IColumn& column) - : _data(assert_cast(column).get_data()) {} + explicit DateV2ParquetConsumer(IColumn& column, ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { DORIS_CHECK_EQ(value_width, sizeof(int32_t)); @@ -54,8 +54,17 @@ class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { _data.resize(old_size + num_values); for (size_t row = 0; row < num_values; ++row) { DateV2Value value; - value.get_date_from_daynr(unaligned_load(values + row * sizeof(int32_t)) + - date_threshold); + const int64_t day_number = + static_cast(unaligned_load(values + row * sizeof(int32_t))) + + date_threshold; + if (day_number < 0 || !value.get_date_from_daynr(static_cast(day_number))) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = DateV2Value(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError("Parquet DATE value is out of range"); + } _data[old_size + row] = value; } return Status::OK(); @@ -63,6 +72,7 @@ class DateV2ParquetConsumer final : public ParquetFixedValueConsumer { private: ColumnDateV2::Container& _data; + ParquetMaterializationState* _state; }; class RejectDateV2BinaryConsumer final : public ParquetBinaryValueConsumer { @@ -202,20 +212,35 @@ Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec context.logical_type != ParquetLogicalType::DATE) { return Status::NotSupported("DATEV2 expects Parquet DATE stored as INT32"); } - DateV2ParquetConsumer consumer(column); + DateV2ParquetConsumer consumer(column, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { return source.decode_fixed_values(num_values, consumer); } if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); - RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DateV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, &state); + RejectDateV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < num_values; ++row) { + if (!state.dictionary_conversion_failures.empty() && + state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + state.mark_conversion_failure(old_size + row); + } + } + } return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index 800febe54288f4..7578661c16f8dd 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -218,11 +218,13 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, using NativeType = typename FieldType::NativeType; DecimalParquetConsumer(IColumn& column, const ParquetDecodeContext& context, - UInt32 target_precision, int32_t target_scale) + UInt32 target_precision, int32_t target_scale, + ParquetMaterializationState* state = nullptr) : _data(assert_cast&>(column).get_data()), _context(context), _target_precision(target_precision), - _target_scale(target_scale) {} + _target_scale(target_scale), + _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { if (_context.physical_type == ParquetPhysicalType::INT32) { @@ -244,6 +246,10 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, auto status = append_binary_value(values + row * value_width, value_width, old_size + row); if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } _data.resize(old_size); return status; } @@ -258,6 +264,10 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, auto status = append_binary_value(reinterpret_cast(values[row].data), values[row].size, old_size + row); if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } _data.resize(old_size); return status; } @@ -274,6 +284,10 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, const auto source_value = unaligned_load(values + row * sizeof(SourceType)); auto status = append_native_value(NativeType(source_value), old_size + row); if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } _data.resize(old_size); return status; } @@ -305,6 +319,7 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, const ParquetDecodeContext& _context; UInt32 _target_precision; int32_t _target_scale; + ParquetMaterializationState* _state; }; } // namespace @@ -677,7 +692,7 @@ Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, if (context.logical_type != ParquetLogicalType::DECIMAL || context.decimal_scale < 0) { return Status::NotSupported("Decimal SerDe requires Parquet DECIMAL metadata"); } - DecimalParquetConsumer consumer(column, context, cast_set(precision), scale); + DecimalParquetConsumer consumer(column, context, cast_set(precision), scale, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { if (context.physical_type == ParquetPhysicalType::BYTE_ARRAY) { return source.decode_binary_values(num_values, consumer); @@ -687,15 +702,29 @@ Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); - RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + DecimalParquetConsumer dictionary_consumer(*state.typed_dictionary, context, + cast_set(precision), scale, &state); + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, dictionary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < num_values; ++row) { + if (!state.dictionary_conversion_failures.empty() && + state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + state.mark_conversion_failure(old_size + row); + } + } + } return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 398b0baff34ab2..f3fcc4978ad4e0 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -201,7 +201,8 @@ Status read_integer_decoded_values(IColumn& column, const DecodedColumnView& vie template Status append_parquet_number(PaddedPODArray& data, const uint8_t* values, - size_t num_values, const ParquetDecodeContext& context) { + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { const size_t old_size = data.size(); data.resize(old_size + num_values); if constexpr (std::is_same_v) { @@ -214,6 +215,12 @@ Status append_parquet_number(PaddedPODArray& data, const uint8_t* for (size_t row = 0; row < num_values; ++row) { const auto value = unaligned_load(values + row * sizeof(SourceType)); if (!decoded_number_value_fits(value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK_LT(old_size + row, state->conversion_failure_null_map->size()); + (*state->conversion_failure_null_map)[old_size + row] = 1; + continue; + } data.resize(old_size); return Status::DataQualityError("Parquet value is out of range at row {}", row); } @@ -224,13 +231,19 @@ Status append_parquet_number(PaddedPODArray& data, const uint8_t* template Status append_parquet_logical_integers(PaddedPODArray& data, const uint8_t* values, - size_t num_values) { + size_t num_values, ParquetMaterializationState* state) { const size_t old_size = data.size(); data.resize(old_size + num_values); for (size_t row = 0; row < num_values; ++row) { const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); const auto logical_value = static_cast(physical_value); if (!decoded_number_value_fits(logical_value)) { + if (state != nullptr && state->can_insert_null_on_conversion_failure()) { + data[old_size + row] = DorisCppType(); + DORIS_CHECK_LT(old_size + row, state->conversion_failure_null_map->size()); + (*state->conversion_failure_null_map)[old_size + row] = 1; + continue; + } data.resize(old_size); return Status::DataQualityError("Parquet logical integer is out of range at row {}", row); @@ -242,24 +255,26 @@ Status append_parquet_logical_integers(PaddedPODArray& data, const template Status append_parquet_integers(PaddedPODArray& data, const uint8_t* values, - size_t num_values, const ParquetDecodeContext& context) { + size_t num_values, const ParquetDecodeContext& context, + ParquetMaterializationState* state) { if (context.logical_integer_bit_width <= 0) { - return append_parquet_number(data, values, num_values, context); + return append_parquet_number(data, values, num_values, context, + state); } if (context.logical_integer_is_signed) { switch (context.logical_integer_bit_width) { case 8: - return append_parquet_logical_integers(data, values, - num_values); + return append_parquet_logical_integers( + data, values, num_values, state); case 16: - return append_parquet_logical_integers(data, values, - num_values); + return append_parquet_logical_integers( + data, values, num_values, state); case 32: - return append_parquet_logical_integers(data, values, - num_values); + return append_parquet_logical_integers( + data, values, num_values, state); case 64: - return append_parquet_logical_integers(data, values, - num_values); + return append_parquet_logical_integers( + data, values, num_values, state); default: return Status::NotSupported("Unsupported Parquet integer bit width {}", context.logical_integer_bit_width); @@ -268,16 +283,16 @@ Status append_parquet_integers(PaddedPODArray& data, const uint8_t switch (context.logical_integer_bit_width) { case 8: return append_parquet_logical_integers(data, values, - num_values); + num_values, state); case 16: return append_parquet_logical_integers(data, values, - num_values); + num_values, state); case 32: return append_parquet_logical_integers(data, values, - num_values); + num_values, state); case 64: return append_parquet_logical_integers(data, values, - num_values); + num_values, state); default: return Status::NotSupported("Unsupported Parquet integer bit width {}", context.logical_integer_bit_width); @@ -290,8 +305,11 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { using DorisCppType = typename PrimitiveTypeTraits::CppType; using ColumnType = typename PrimitiveTypeTraits::ColumnType; - NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context) - : _data(assert_cast(column).get_data()), _context(context) {} + NumberParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { return consume_impl(values, num_values, value_width); @@ -319,6 +337,12 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { const float value = parquet_half_to_float( unaligned_load(values + row * sizeof(uint16_t))); if (!decoded_number_value_fits(value)) { + if (_state != nullptr && _state->can_insert_null_on_conversion_failure()) { + _data[old_size + row] = DorisCppType(); + DORIS_CHECK_LT(old_size + row, _state->conversion_failure_null_map->size()); + (*_state->conversion_failure_null_map)[old_size + row] = 1; + continue; + } _data.resize(old_size); return Status::DataQualityError( "Parquet FLOAT16 value is out of range at row {}", row); @@ -330,22 +354,24 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { switch (_context.physical_type) { case ParquetPhysicalType::BOOLEAN: DORIS_CHECK_EQ(value_width, sizeof(uint8_t)); - return append_parquet_number(_data, values, num_values, - _context); + return append_parquet_number(_data, values, num_values, _context, + _state); case ParquetPhysicalType::INT32: DORIS_CHECK_EQ(value_width, sizeof(int32_t)); return append_parquet_integers(_data, values, num_values, - _context); + _context, _state); case ParquetPhysicalType::INT64: DORIS_CHECK_EQ(value_width, sizeof(int64_t)); return append_parquet_integers(_data, values, num_values, - _context); + _context, _state); case ParquetPhysicalType::FLOAT: DORIS_CHECK_EQ(value_width, sizeof(float)); - return append_parquet_number(_data, values, num_values, _context); + return append_parquet_number(_data, values, num_values, _context, + _state); case ParquetPhysicalType::DOUBLE: DORIS_CHECK_EQ(value_width, sizeof(double)); - return append_parquet_number(_data, values, num_values, _context); + return append_parquet_number(_data, values, num_values, _context, + _state); default: return Status::NotSupported("Unsupported Parquet physical type {} for numeric SerDe", static_cast(_context.physical_type)); @@ -354,6 +380,7 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { PaddedPODArray& _data; const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; }; class RejectParquetBinaryConsumer final : public ParquetBinaryValueConsumer { @@ -539,21 +566,37 @@ Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, T == TYPE_DOUBLE)) { return DataTypeSerDe::read_column_from_parquet(column, source, context, num_values, state); } else { - NumberParquetConsumer consumer(column, context); + NumberParquetConsumer consumer(column, context, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { return source.decode_fixed_values(num_values, consumer); } if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); - RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + NumberParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectParquetBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure() && + !state.dictionary_conversion_failures.empty()) { + for (size_t row = 0; row < num_values; ++row) { + if (state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + DORIS_CHECK_LT(old_size + row, state.conversion_failure_null_map->size()); + (*state.conversion_failure_null_map)[old_size + row] = 1; + } + } + } return Status::OK(); } } diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 0d2d6c4034234e..b9f170cb95b8ff 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -68,18 +68,28 @@ class StringParquetConsumer final : public ParquetFixedValueConsumer, explicit StringParquetConsumer(IColumn& column) : _column(assert_cast(column)) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { - static constexpr size_t BATCH_SIZE = 256; - std::array refs; - size_t offset = 0; - while (offset < num_values) { - const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); - for (size_t row = 0; row < batch_size; ++row) { - refs[row] = StringRef( - reinterpret_cast(values + (offset + row) * value_width), - value_width); + if constexpr (requires(ColumnType& column) { + column.insert_many_fixed_length_data(static_cast(nullptr), + size_t(), size_t()); + }) { + // FIXED_LEN_BYTE_ARRAY is already a dense byte span. Copy it once and synthesize + // offsets; StringRef batches add a second row loop and hundreds of tiny memcpy calls. + _column.insert_many_fixed_length_data(reinterpret_cast(values), + value_width, num_values); + } else { + static constexpr size_t BATCH_SIZE = 256; + std::array refs; + size_t offset = 0; + while (offset < num_values) { + const size_t batch_size = std::min(BATCH_SIZE, num_values - offset); + for (size_t row = 0; row < batch_size; ++row) { + refs[row] = StringRef( + reinterpret_cast(values + (offset + row) * value_width), + value_width); + } + _column.insert_many_strings(refs.data(), batch_size); + offset += batch_size; } - _column.insert_many_strings(refs.data(), batch_size); - offset += batch_size; } return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index 8c22f8a1ae336b..793be706b4aaf5 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -17,6 +17,8 @@ #include "core/data_type_serde/data_type_time_serde.h" +#include + #include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_number.h" #include "core/data_type/primitive_type.h" @@ -55,8 +57,11 @@ TimeValue::TimeType read_time_decoded_value(const DecodedColumnView& view, int64 class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { public: - TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context) - : _data(assert_cast(column).get_data()), _context(context) {} + TimeV2ParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { const size_t old_size = _data.size(); @@ -73,6 +78,16 @@ class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { DORIS_CHECK_EQ(value_width, sizeof(int64_t)); micros = unaligned_load(values + row * sizeof(int64_t)); if (_context.time_unit == ParquetTimeUnit::MILLIS) { + if (micros > std::numeric_limits::max() / 1000 || + micros < std::numeric_limits::min() / 1000) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = TimeValue::TimeType(); + continue; + } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet TIME value overflows microseconds"); + } micros *= 1000; } else if (_context.time_unit == ParquetTimeUnit::NANOS) { micros /= 1000; @@ -94,6 +109,7 @@ class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { private: ColumnTimeV2::Container& _data; const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; }; class RejectTimeV2BinaryConsumer final : public ParquetBinaryValueConsumer { @@ -261,20 +277,35 @@ Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec context.logical_type != ParquetLogicalType::TIME) { return Status::NotSupported("TIMEV2 expects Parquet TIME stored as INT32 or INT64"); } - TimeV2ParquetConsumer consumer(column, context); + TimeV2ParquetConsumer consumer(column, context, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { return source.decode_fixed_values(num_values, consumer); } if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); - RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimeV2ParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimeV2BinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < num_values; ++row) { + if (!state.dictionary_conversion_failures.empty() && + state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + state.mark_conversion_failure(old_size + row); + } + } + } return Status::OK(); } diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index 42a95cdda7a183..ec6e9e6941d876 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -212,12 +212,45 @@ struct ParquetMaterializationState { std::vector dictionary_indices; ParquetSelection selection; uint64_t dictionary_generation = std::numeric_limits::max(); + bool enable_strict_mode = false; + IColumn::Filter* conversion_failure_null_map = nullptr; + IColumn::Filter dictionary_conversion_failures; void reset_dictionary() { typed_dictionary.reset(); dictionary_indices.clear(); + dictionary_conversion_failures.clear(); dictionary_generation = std::numeric_limits::max(); } + + bool can_insert_null_on_conversion_failure() const { + return !enable_strict_mode && conversion_failure_null_map != nullptr; + } + + bool mark_conversion_failure(size_t output_row) { + if (!can_insert_null_on_conversion_failure()) { + return false; + } + DORIS_CHECK_LT(output_row, conversion_failure_null_map->size()); + (*conversion_failure_null_map)[output_row] = 1; + return true; + } + + IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { + auto* output_null_map = conversion_failure_null_map; + dictionary_conversion_failures.clear(); + if (can_insert_null_on_conversion_failure()) { + // Only nullable non-strict outputs may absorb a bad dictionary entry. Redirecting a + // non-nullable decode here would silently turn a required error into a default value. + dictionary_conversion_failures.resize_fill(dictionary_size, 0); + conversion_failure_null_map = &dictionary_conversion_failures; + } + return output_null_map; + } + + void end_dictionary_conversion(IColumn::Filter* output_null_map) { + conversion_failure_null_map = output_null_map; + } }; } // namespace doris diff --git a/be/src/format/parquet/parquet_thrift_util.h b/be/src/format/parquet/parquet_thrift_util.h index 1dbbf8eff20ffd..d28b982fea5911 100644 --- a/be/src/format/parquet/parquet_thrift_util.h +++ b/be/src/format/parquet/parquet_thrift_util.h @@ -39,7 +39,8 @@ constexpr size_t INIT_META_SIZE = 48 * 1024; // 48k static Status parse_thrift_footer(io::FileReaderSPtr file, std::unique_ptr* file_metadata, size_t* meta_size, io::IOContext* io_ctx, const bool enable_mapping_varbinary, - const bool enable_mapping_timestamp_tz) { + const bool enable_mapping_timestamp_tz, + bool retain_serialized_metadata = false) { size_t file_size = file->size(); size_t bytes_read = std::min(file_size, INIT_META_SIZE); std::vector footer(bytes_read); @@ -77,10 +78,17 @@ static Status parse_thrift_footer(io::FileReaderSPtr file, meta_ptr = footer.data() + bytes_read - PARQUET_FOOTER_SIZE - metadata_size; } + std::vector serialized_metadata; + if (retain_serialized_metadata) { + // Small-file v2 opens still need Arrow's page-index adapter. Retaining the bytes already + // fetched from storage avoids serializing the just-decoded Thrift footer a second time. + serialized_metadata.assign(meta_ptr, meta_ptr + metadata_size); + } tparquet::FileMetaData t_metadata; // deserialize footer RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata)); - *file_metadata = std::make_unique(t_metadata, metadata_size); + *file_metadata = std::make_unique(t_metadata, metadata_size, + std::move(serialized_metadata)); RETURN_IF_ERROR( (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); *meta_size = PARQUET_FOOTER_SIZE + metadata_size; diff --git a/be/src/format/parquet/vparquet_file_metadata.cpp b/be/src/format/parquet/vparquet_file_metadata.cpp index b204ee24186ae7..c39a84e1715ca9 100644 --- a/be/src/format/parquet/vparquet_file_metadata.cpp +++ b/be/src/format/parquet/vparquet_file_metadata.cpp @@ -32,13 +32,17 @@ namespace doris { -FileMetaData::FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size) - : _metadata(metadata), _mem_size(mem_size) { - ExecEnv::GetInstance()->parquet_meta_tracker()->consume(mem_size); +FileMetaData::FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size, + std::vector serialized_metadata) + : _metadata(metadata), + _mem_size(mem_size), + _serialized_metadata(std::move(serialized_metadata)) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(mem_size + _serialized_metadata.size()); } FileMetaData::~FileMetaData() { - ExecEnv::GetInstance()->parquet_meta_tracker()->release(_mem_size + _arrow_metadata_mem_size); + ExecEnv::GetInstance()->parquet_meta_tracker()->release( + _mem_size + _serialized_metadata.size() + _arrow_metadata_mem_size); } Status FileMetaData::init_schema(const bool enable_mapping_varbinary, @@ -63,16 +67,21 @@ Status FileMetaData::get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData> std::lock_guard lock(_arrow_metadata_mutex); if (_arrow_metadata == nullptr) { try { - ThriftSerializer serializer(/*compact=*/true, - static_cast(std::max(_mem_size, 4096))); - std::vector serialized_metadata; - RETURN_IF_ERROR(serializer.serialize(const_cast(&_metadata), - &serialized_metadata)); - uint32_t serialized_size = cast_set(serialized_metadata.size()); + std::vector fallback_serialized_metadata; + const std::vector* serialized_metadata = &_serialized_metadata; + if (serialized_metadata->empty()) { + ThriftSerializer serializer(/*compact=*/true, + static_cast(std::max(_mem_size, 4096))); + RETURN_IF_ERROR( + serializer.serialize(const_cast(&_metadata), + &fallback_serialized_metadata)); + serialized_metadata = &fallback_serialized_metadata; + } + uint32_t serialized_size = cast_set(serialized_metadata->size()); auto parsed = - ::parquet::FileMetaData::Make(serialized_metadata.data(), &serialized_size, + ::parquet::FileMetaData::Make(serialized_metadata->data(), &serialized_size, ::parquet::default_reader_properties()); - DORIS_CHECK(static_cast(serialized_size) == serialized_metadata.size()); + DORIS_CHECK(static_cast(serialized_size) == serialized_metadata->size()); // Native and Arrow planners describe the same immutable footer. Cache the adapter at // the footer-cache lifecycle so repeated v2 opens do not serialize and parse it again. _arrow_metadata_mem_size = parsed->size(); diff --git a/be/src/format/parquet/vparquet_file_metadata.h b/be/src/format/parquet/vparquet_file_metadata.h index 161faa9184f503..8089e20f1a40be 100644 --- a/be/src/format/parquet/vparquet_file_metadata.h +++ b/be/src/format/parquet/vparquet_file_metadata.h @@ -21,6 +21,7 @@ #include #include #include +#include #include "common/status.h" #include "format/parquet/schema_desc.h" @@ -32,7 +33,8 @@ class FileMetaData; namespace doris { class FileMetaData { public: - FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size); + FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size, + std::vector serialized_metadata = {}); ~FileMetaData(); Status init_schema(const bool enable_mapping_varbinary, const bool enable_mapping_timestamp_tz); const FieldDescriptor& schema() const { return _schema; } @@ -40,7 +42,7 @@ class FileMetaData { const tparquet::FileMetaData& to_thrift() const; Status get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const; std::string debug_string() const; - size_t get_mem_size() const { return _mem_size; } + size_t get_mem_size() const { return _mem_size + _serialized_metadata.size(); } private: tparquet::FileMetaData _metadata; @@ -49,6 +51,7 @@ class FileMetaData { mutable std::mutex _arrow_metadata_mutex; mutable std::shared_ptr<::parquet::FileMetaData> _arrow_metadata; mutable size_t _arrow_metadata_mem_size = 0; + std::vector _serialized_metadata; }; } // namespace doris diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 3957637962dd53..d7bbc39245d4da 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -34,6 +34,7 @@ #include "common/check.h" #include "common/config.h" #include "format/parquet/parquet_thrift_util.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" #include "io/fs/buffered_reader.h" @@ -595,7 +596,8 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont } else { RETURN_IF_ERROR(parse_thrift_footer( native_file, &native_metadata_owner, &native_footer_size, io_ctx, - /*enable_mapping_varbinary=*/true, enable_mapping_timestamp_tz)); + /*enable_mapping_varbinary=*/true, enable_mapping_timestamp_tz, + /*retain_serialized_metadata=*/true)); ++native_footer_read_calls; if (meta_cache != nullptr && meta_cache->enabled()) { meta_cache->insert(meta_cache_key, native_metadata_owner.release(), @@ -648,6 +650,13 @@ Status ParquetFileContext::load_native_offset_indexes( if (leaf_column_ids.empty() || file_reader == nullptr) { return Status::OK(); } + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for OffsetIndex", row_group_id); + } + const auto& native_row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); try { auto page_index_reader = file_reader->GetPageIndexReader(); if (page_index_reader == nullptr) { @@ -658,6 +667,11 @@ Status ParquetFileContext::load_native_offset_indexes( return Status::OK(); } for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(native_row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for OffsetIndex", + leaf_column_id); + } auto arrow_index = row_group_reader->GetOffsetIndex(leaf_column_id); if (arrow_index == nullptr || arrow_index->page_locations().empty()) { // An empty optional index is equivalent to no index. Publishing it would select @@ -673,6 +687,16 @@ Status ParquetFileContext::load_native_offset_indexes( native_location.__set_first_row_index(arrow_location.first_row_index); native_index.page_locations.push_back(std::move(native_location)); } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + native_row_group.columns[leaf_column_id].meta_data, native_file->size(), + compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index(native_index, chunk_range, + native_row_group.num_rows)) { + // OffsetIndex is optional. Reject the complete index instead of letting one bad + // location redirect an indexed reader outside its owning column chunk. + continue; + } offset_indexes->emplace(leaf_column_id, std::move(native_index)); } } catch (const ::parquet::ParquetException&) { diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp index 761a7f6a591f8a..ee4955364110d5 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -134,10 +134,12 @@ Status CountColumnReader::create(io::FileReaderSPtr file, const FileMetaData* me const size_t max_group_buffer = config::parquet_rowgroup_max_buffer_mb << 20; const size_t max_column_buffer = config::parquet_column_max_buffer_mb << 20; std::unique_ptr level_reader; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); RETURN_IF_ERROR(native::LevelReader::create( std::move(file), row_group.columns[leaf_schema->leaf_column_id], leaf_field, row_group.num_rows, std::min(max_group_buffer, max_column_buffer), io_ctx, - enable_page_cache, page_cache_file_key, &level_reader)); + enable_page_cache, page_cache_file_key, compat, &level_reader)); reader->reset(new CountColumnReader(leaf_schema->name, std::move(level_reader), profile)); return Status::OK(); } diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 2c5a04dd5cc447..7b29bfcad70272 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -19,9 +19,12 @@ #include #include +#include #include +#include #include +#include #include #include @@ -50,6 +53,77 @@ struct IOContext; } // namespace doris namespace doris::format::parquet::native { + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { + if (created_by.empty()) { + return {}; + } + const ::parquet::ApplicationVersion version(created_by); + return {.parquet_816_padding = + version.VersionLt(::parquet::ApplicationVersion::PARQUET_816_FIXED_VERSION()), + .data_page_v2_always_compressed = version.VersionLt( + ::parquet::ApplicationVersion::PARQUET_CPP_10353_FIXED_VERSION())}; +} + +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range) { + DORIS_CHECK(range != nullptr); + int64_t start = metadata.data_page_offset; + if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset >= 0 && + metadata.dictionary_page_offset < start) { + start = metadata.dictionary_page_offset; + } + const int64_t length = metadata.total_compressed_size; + if (UNLIKELY(start < 0 || length < 0)) { + return Status::Corruption("Parquet column chunk has a negative offset or length"); + } + const uint64_t unsigned_start = static_cast(start); + const uint64_t unsigned_length = static_cast(length); + if (UNLIKELY(unsigned_start > file_size || unsigned_length > file_size - unsigned_start)) { + // Thrift range fields are signed and untrusted; validate before converting them to the + // unsigned stream-reader coordinates so overflow cannot wrap back into the file. + return Status::Corruption("Parquet column chunk [{}, {}) exceeds file size {}", start, + unsigned_start + unsigned_length, file_size); + } + size_t bounded_length = static_cast(unsigned_length); + if (parquet_816_padding) { + // parquet-mr before PARQUET-816 under-reported the chunk by up to 100 bytes. Padding stays + // file-bounded and is only enabled for the affected writer versions. + bounded_length += std::min(100, file_size - unsigned_start - unsigned_length); + } + range->offset = static_cast(unsigned_start); + range->length = bounded_length; + return Status::OK(); +} + +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t row_count) { + if (index.page_locations.empty() || row_count < 0 || + index.page_locations.front().first_row_index != 0 || + chunk_range.length > std::numeric_limits::max() - chunk_range.offset) { + return false; + } + const uint64_t chunk_begin = chunk_range.offset; + const uint64_t chunk_end = chunk_begin + chunk_range.length; + uint64_t previous_end = chunk_begin; + int64_t previous_row = -1; + for (const auto& location : index.page_locations) { + if (location.first_row_index <= previous_row || location.first_row_index >= row_count || + location.offset < 0 || location.compressed_page_size <= 0) { + return false; + } + const uint64_t begin = static_cast(location.offset); + const uint64_t size = static_cast(location.compressed_page_size); + if (begin < chunk_begin || begin < previous_end || begin > chunk_end || + size > chunk_end - begin) { + return false; + } + previous_row = location.first_row_index; + previous_end = begin + size; + } + return true; +} + namespace { Status translate_value_encoding(tparquet::Encoding::type encoding, @@ -193,7 +267,8 @@ template ColumnChunkReader::ColumnChunkReader( io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows, - io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx) + io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range) : _field_schema(field_schema), _max_rep_level(field_schema->repetition_level), _max_def_level(field_schema->definition_level), @@ -202,13 +277,21 @@ ColumnChunkReader::ColumnChunkReader( _offset_index(offset_index), _total_rows(total_rows), _io_ctx(io_ctx), - _page_read_ctx(page_read_ctx) {} + _page_read_ctx(page_read_ctx) { + if (chunk_range != nullptr) { + _chunk_range = *chunk_range; + _has_validated_chunk_range = true; + } +} template Status ColumnChunkReader::init() { - size_t start_offset = has_dict_page(_metadata) ? _metadata.dictionary_page_offset - : _metadata.data_page_offset; - size_t chunk_size = _metadata.total_compressed_size; + size_t start_offset = _has_validated_chunk_range + ? _chunk_range.offset + : (has_dict_page(_metadata) ? _metadata.dictionary_page_offset + : _metadata.data_page_offset); + size_t chunk_size = + _has_validated_chunk_range ? _chunk_range.length : _metadata.total_compressed_size; // create page reader _page_reader = create_page_reader( _stream_reader, _io_ctx, start_offset, chunk_size, _total_rows, _metadata, @@ -281,20 +364,29 @@ Status ColumnChunkReader::read_levels( template Status ColumnChunkReader::_parse_first_page_header() { - RETURN_IF_ERROR(parse_page_header()); - - const tparquet::PageHeader* header = nullptr; - RETURN_IF_ERROR(_page_reader->get_page_header(&header)); - if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + const tparquet::PageHeader* header = nullptr; + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + _state = INITIALIZED; + return parse_page_header(); + } + if (header->type != tparquet::PageType::DICTIONARY_PAGE) { + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + _state = INITIALIZED; + continue; + } // the first page maybe directory page even if _metadata.__isset.dictionary_page_offset == false, // so we should parse the directory page in next_page() RETURN_IF_ERROR(_decode_dict_page()); // parse the real first data page RETURN_IF_ERROR(_page_reader->dict_next_page()); _state = INITIALIZED; + // A dictionary is the only non-data page with decoder state. Any following index or + // extension pages are skipped by the same pre-data loop. } - - return Status::OK(); } template @@ -302,10 +394,19 @@ Status ColumnChunkReader::parse_page_header() { if (_state == HEADER_PARSED || _state == DATA_LOADED) { return Status::OK(); } - RETURN_IF_ERROR(_page_reader->parse_page_header()); - const tparquet::PageHeader* header = nullptr; - RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + while (true) { + RETURN_IF_ERROR(_page_reader->parse_page_header()); + RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + if (header->type == tparquet::PageType::DATA_PAGE || + header->type == tparquet::PageType::DATA_PAGE_V2) { + break; + } + if (header->type == tparquet::PageType::DICTIONARY_PAGE) { + return Status::Corruption("Parquet dictionary page appears after data pages"); + } + RETURN_IF_ERROR(_page_reader->skip_auxiliary_page()); + } int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values : header->data_page_header.num_values; _remaining_rep_nums = page_num_values; @@ -446,7 +547,8 @@ Status ColumnChunkReader::load_page_data() { RETURN_IF_ERROR(_get_uncompressed_levels(header_v2, compressed_data)); } bool is_v2_compressed = header->__isset.data_page_header_v2 && - header->data_page_header_v2.is_compressed; + (header->data_page_header_v2.is_compressed || + _page_read_ctx.data_page_v2_always_compressed); bool page_has_compression = header->__isset.data_page_header || is_v2_compressed; if (page_has_compression) { @@ -462,7 +564,8 @@ Status ColumnChunkReader::load_page_data() { } // Decide whether to cache decompressed payload or compressed payload based on threshold - bool cache_payload_decompressed = should_cache_decompressed(header, _metadata); + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && @@ -662,7 +765,8 @@ Status ColumnChunkReader::_decode_dict_page() { // Decide whether to cache decompressed or compressed dictionary based on threshold // If uncompressed_page_size == 0, should_cache_decompressed will return true - bool cache_payload_decompressed = should_cache_decompressed(header, _metadata); + bool cache_payload_decompressed = should_cache_decompressed( + header, _metadata, _page_read_ctx.data_page_v2_always_compressed); if (_page_read_ctx.enable_parquet_file_page_cache && !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr && diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 6e71ae52a6a6a4..ec27e5e18c61e6 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,22 @@ namespace doris::format::parquet::native { using ::doris::FieldSchema; using ::doris::ColumnString; +struct ColumnChunkRange { + size_t offset = 0; + size_t length = 0; +}; + +struct ParquetReaderCompat { + bool parquet_816_padding = false; + bool data_page_v2_always_compressed = false; +}; + +ParquetReaderCompat parquet_reader_compat(const std::string& created_by); +Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, + bool parquet_816_padding, ColumnChunkRange* range); +bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, + int64_t row_count); + struct ColumnChunkReaderStatistics { int64_t decompress_time = 0; int64_t decompress_cnt = 0; @@ -102,7 +119,8 @@ class ColumnChunkReader { ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_row, io::IOContext* io_ctx, - const ParquetPageReadContext& page_read_ctx); + const ParquetPageReadContext& page_read_ctx, + const ColumnChunkRange* chunk_range = nullptr); ~ColumnChunkReader() = default; // Initialize chunk reader, will generate the decoder and codec. @@ -296,6 +314,8 @@ class ColumnChunkReader { BlockCompressionCodec* _block_compress_codec = nullptr; ParquetPageReadContext _page_read_ctx; + ColumnChunkRange _chunk_range; + bool _has_validated_chunk_range = false; LevelDecoder _rep_level_decoder; LevelDecoder _def_level_decoder; diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 7a7b72657bcfba..579f0f213f3a63 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -356,13 +356,14 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, RuntimeState* state, bool in_collection, const std::set& column_ids, const std::set& filter_column_ids, - const std::string& page_cache_file_key) { + const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode) { size_t total_rows = row_group.num_rows; if (field->data_type->get_primitive_type() == TYPE_ARRAY) { std::unique_ptr element_reader; RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, element_reader, max_buf_size, col_offsets, state, true, column_ids, - filter_column_ids, page_cache_file_key)); + filter_column_ids, page_cache_file_key, compat, enable_strict_mode)); auto array_reader = ArrayColumnReader::create_unique(row_ranges, total_rows, ctz, io_ctx); element_reader->set_column_in_nested(); RETURN_IF_ERROR(array_reader->init(std::move(element_reader), field)); @@ -377,7 +378,8 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, // Create key reader RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, key_reader, max_buf_size, col_offsets, state, true, column_ids, - filter_column_ids, page_cache_file_key)); + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); } else { auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, io_ctx, &field->children[0]); @@ -389,7 +391,8 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, // Create value reader RETURN_IF_ERROR(create(file, &field->children[1], row_group, row_ranges, ctz, io_ctx, value_reader, max_buf_size, col_offsets, state, true, column_ids, - filter_column_ids, page_cache_file_key)); + filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); } else { auto skip_reader = std::make_unique(row_ranges, total_rows, ctz, io_ctx, &field->children[1]); @@ -413,7 +416,7 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, RETURN_IF_ERROR(create(file, &child, row_group, row_ranges, ctz, io_ctx, child_reader, max_buf_size, col_offsets, state, in_collection, column_ids, filter_column_ids, - page_cache_file_key)); + page_cache_file_key, compat, enable_strict_mode)); child_readers[child.name] = std::move(child_reader); // Record the first non-SkippingReader if (non_skip_reader_idx == -1) { @@ -432,7 +435,8 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, std::unique_ptr child_reader; RETURN_IF_ERROR(create(file, &field->children[0], row_group, row_ranges, ctz, io_ctx, child_reader, max_buf_size, col_offsets, state, in_collection, - column_ids, filter_column_ids, page_cache_file_key)); + column_ids, filter_column_ids, page_cache_file_key, compat, + enable_strict_mode)); child_reader->set_column_in_nested(); child_readers[field->children[0].name] = std::move(child_reader); } @@ -452,16 +456,18 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR( - scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } else { auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR( - scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } @@ -470,16 +476,18 @@ Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR( - scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } else { auto scalar_reader = ScalarColumnReader::create_unique( row_ranges, total_rows, chunk, offset_index, ctz, io_ctx); - RETURN_IF_ERROR( - scalar_reader->init(file, field, max_buf_size, state, page_cache_file_key)); + RETURN_IF_ERROR(scalar_reader->init(file, field, max_buf_size, state, + page_cache_file_key, compat, + enable_strict_mode)); scalar_reader->_filter_column_ids = filter_column_ids; reader.reset(scalar_reader.release()); } @@ -496,12 +504,15 @@ void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* res template Status ScalarColumnReader::init( io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, RuntimeState* state, - const std::string& page_cache_file_key) { + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, + bool enable_strict_mode) { _field_schema = field; auto& chunk_meta = _chunk_meta.meta_data; - int64_t chunk_start = has_dict_page(chunk_meta) ? chunk_meta.dictionary_page_offset - : chunk_meta.data_page_offset; - size_t chunk_len = chunk_meta.total_compressed_size; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(chunk_meta, file->size(), compat.parquet_816_padding, + &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_len = chunk_range.length; size_t prefetch_buffer_size = std::min(chunk_len, max_buf_size); if ((typeid_cast(file.get()) && typeid_cast( @@ -514,10 +525,12 @@ Status ScalarColumnReader::init( prefetch_buffer_size); ParquetPageReadContext ctx( (state == nullptr) ? true : state->query_options().enable_parquet_file_page_cache, - page_cache_file_key); + page_cache_file_key, compat.data_page_v2_always_compressed); _chunk_reader = std::make_unique>( - _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx); + _stream_reader.get(), &_chunk_meta, field, _offset_index, _total_rows, _io_ctx, ctx, + &chunk_range); + _materialization_state.enable_strict_mode = enable_strict_mode; RETURN_IF_ERROR(_chunk_reader->init()); RETURN_IF_ERROR(init_decode_context(*field, _ctz, &_decode_context)); return Status::OK(); @@ -717,6 +730,9 @@ Status ScalarColumnReader::_read_values(size_t num_ _filter_map_index += num_values; } DORIS_CHECK(_serde != nullptr); + // Keep selected-row cardinality stable: non-strict conversion failures append a nested + // default and mark this matching nullable output row instead of shortening the column. + _materialization_state.conversion_failure_null_map = map_data_column; return _chunk_reader->materialize_values(data_column, *_serde, _decode_context, _materialization_state, _select_vector); } @@ -779,6 +795,8 @@ Status ScalarColumnReader::_read_nested_column( } DORIS_CHECK(_serde != nullptr); + // Nested materialization must preserve the same value/null-map row alignment invariant. + _materialization_state.conversion_failure_null_map = map_data_column; RETURN_IF_ERROR(_chunk_reader->materialize_values(data_column, *_serde, _decode_context, _materialization_state, _select_vector)); if (!_ancestor_null_indices.empty()) { diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 19f78a18f1e0ed..2b70a416f07c88 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -206,7 +206,8 @@ class ColumnReader { RuntimeState* state, bool in_collection = false, const std::set& column_ids = {}, const std::set& filter_column_ids = {}, - const std::string& page_cache_file_key = {}); + const std::string& page_cache_file_key = {}, + const ParquetReaderCompat& compat = {}, bool enable_strict_mode = false); virtual const std::vector& get_rep_level() const = 0; virtual const std::vector& get_def_level() const = 0; virtual ColumnStatistics column_statistics() = 0; @@ -254,7 +255,8 @@ class ScalarColumnReader : public ColumnReader { _offset_index(offset_index) {} ~ScalarColumnReader() override { close(); } Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, - RuntimeState* state, const std::string& page_cache_file_key); + RuntimeState* state, const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, bool enable_strict_mode); Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp index 1f28a4a3f61ca6..4b9aa202cd4f97 100644 --- a/be/src/format_v2/parquet/reader/native/level_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -42,7 +42,8 @@ class LevelReaderImpl final : public LevelReader::Impl { public: LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) + bool enable_page_cache, std::string page_cache_file_key, + ParquetReaderCompat compat) : _file(std::move(file)), _column_chunk(std::move(column_chunk)), _field(field), @@ -50,15 +51,18 @@ class LevelReaderImpl final : public LevelReader::Impl { _max_buffer_size(max_buffer_size), _io_ctx(io_ctx), _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)) {} + _page_cache_file_key(std::move(page_cache_file_key)), + _compat(compat) {} Status init() override { DORIS_CHECK(_file != nullptr); DORIS_CHECK(_field != nullptr); const auto& metadata = _column_chunk.meta_data; - const int64_t chunk_start = has_dict_page(metadata) ? metadata.dictionary_page_offset - : metadata.data_page_offset; - const size_t chunk_size = metadata.total_compressed_size; + ColumnChunkRange chunk_range; + RETURN_IF_ERROR(compute_column_chunk_range(metadata, _file->size(), + _compat.parquet_816_padding, &chunk_range)); + const size_t chunk_start = chunk_range.offset; + const size_t chunk_size = chunk_range.length; size_t prefetch_buffer_size = std::min(chunk_size, _max_buffer_size); auto* tracing_reader = typeid_cast(_file.get()); if ((tracing_reader != nullptr && @@ -71,7 +75,9 @@ class LevelReaderImpl final : public LevelReader::Impl { prefetch_buffer_size); _chunk_reader = std::make_unique>( _stream.get(), &_column_chunk, _field, nullptr, _total_rows, _io_ctx, - ParquetPageReadContext(_enable_page_cache, _page_cache_file_key)); + ParquetPageReadContext(_enable_page_cache, _page_cache_file_key, + _compat.data_page_v2_always_compressed), + &chunk_range); return _chunk_reader->init(); } @@ -155,6 +161,7 @@ class LevelReaderImpl final : public LevelReader::Impl { io::IOContext* _io_ctx = nullptr; bool _enable_page_cache = false; std::string _page_cache_file_key; + ParquetReaderCompat _compat; size_t _current_row = 0; std::unique_ptr _stream; std::unique_ptr> _chunk_reader; @@ -164,18 +171,19 @@ Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column FieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, bool enable_page_cache, const std::string& page_cache_file_key, + const ParquetReaderCompat& compat, std::unique_ptr* reader) { DORIS_CHECK(reader != nullptr); DORIS_CHECK(field != nullptr); std::unique_ptr impl; if (field->repetition_level > 0) { - impl = std::make_unique>(std::move(file), std::move(column_chunk), - field, total_rows, max_buffer_size, io_ctx, - enable_page_cache, page_cache_file_key); + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); } else { - impl = std::make_unique>(std::move(file), std::move(column_chunk), - field, total_rows, max_buffer_size, io_ctx, - enable_page_cache, page_cache_file_key); + impl = std::make_unique>( + std::move(file), std::move(column_chunk), field, total_rows, max_buffer_size, + io_ctx, enable_page_cache, page_cache_file_key, compat); } RETURN_IF_ERROR(impl->init()); reader->reset(new LevelReader(std::move(impl))); diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h index b5e929cb1d08cf..814c7ffa0ed804 100644 --- a/be/src/format_v2/parquet/reader/native/level_reader.h +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -52,7 +52,7 @@ class LevelReader { static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, bool enable_page_cache, - const std::string& page_cache_file_key, + const std::string& page_cache_file_key, const ParquetReaderCompat& compat, std::unique_ptr* reader); ~LevelReader(); diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp index 20f585ae021da4..2e503d4f31fe15 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -158,8 +158,8 @@ Status PageReader::parse_page_header() { // Increment page cache counters for a true cache hit on header+payload _page_statistics.page_cache_hit_counter += 1; // Detect whether the cached payload is compressed or decompressed and record - bool is_cache_payload_decompressed = - should_cache_decompressed(&_cur_page_header, _metadata); + bool is_cache_payload_decompressed = should_cache_decompressed( + &_cur_page_header, _metadata, _page_read_ctx.data_page_v2_always_compressed); if (is_cache_payload_decompressed) { _page_statistics.page_cache_decompressed_hit_counter += 1; diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h index 641a047eb38ca8..a798a01f9f4a53 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.h +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -57,18 +57,23 @@ struct ParquetPageReadContext { // A default-constructed context has no stable file identity, so cache lookup must stay off. bool enable_parquet_file_page_cache = false; std::string page_cache_file_key; + bool data_page_v2_always_compressed = false; ParquetPageReadContext() = default; - ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key) + ParquetPageReadContext(bool enable_parquet_file_page_cache, std::string page_cache_file_key, + bool data_page_v2_always_compressed = false) : enable_parquet_file_page_cache(enable_parquet_file_page_cache && !page_cache_file_key.empty()), - page_cache_file_key(std::move(page_cache_file_key)) {} + page_cache_file_key(std::move(page_cache_file_key)), + data_page_v2_always_compressed(data_page_v2_always_compressed) {} }; inline bool should_cache_decompressed(const tparquet::PageHeader* header, - const tparquet::ColumnMetaData& metadata) { + const tparquet::ColumnMetaData& metadata, + bool data_page_v2_always_compressed = false) { // Data Page V2 declares its payload representation independently of the column codec. A warm // hit must never send an explicitly uncompressed cached payload through the codec again. - if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed) { + if (header->__isset.data_page_header_v2 && !header->data_page_header_v2.is_compressed && + !data_page_v2_always_compressed) { return true; } if (header->compressed_page_size <= 0) return true; @@ -170,6 +175,18 @@ class PageReader { return Status::OK(); } + Status skip_auxiliary_page() { + if constexpr (OFFSET_INDEX) { + // OffsetIndex enumerates data pages only, so an auxiliary physical page must not + // consume a logical page-location entry while advancing to its payload end. + skip_page_data(); + _state = INITIALIZED; + return Status::OK(); + } else { + return next_page(); + } + } + Status dict_next_page() { if constexpr (OFFSET_INDEX) { _state = INITIALIZED; @@ -205,7 +222,8 @@ class PageReader { } uint64_t file_end_offset() const { return _end_offset; } bool cached_decompressed() const { - return should_cache_decompressed(&_cur_page_header, _metadata); + return should_cache_decompressed(&_cur_page_header, _metadata, + _page_read_ctx.data_page_v2_always_compressed); } PageStatistics& page_statistics() { return _page_statistics; } diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 68335ee014d861..d4f8510f73021e 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -247,10 +247,14 @@ Status NativeColumnReader::init( _page_cache_runtime_state = RuntimeState::create_unique(query_options, TQueryGlobals()); native_runtime_state = _page_cache_runtime_state.get(); } + const auto& thrift_metadata = metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); RETURN_IF_ERROR(native::ColumnReader::create( std::move(file), field, row_group, _row_ranges, timezone, io_ctx, _native_reader, max_buffer_size, *_offset_indexes, native_runtime_state, false, _projected_column_ids, - _filter_column_ids, page_cache_file_key)); + _filter_column_ids, page_cache_file_key, compat, + runtime_state != nullptr && runtime_state->enable_strict_mode())); DORIS_CHECK(_native_reader != nullptr); _skip_column = _type->create_column(); return Status::OK(); diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp index f4379b2e60e692..80a72670300c37 100644 --- a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -318,5 +319,95 @@ TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { 16)); } +TEST(DataTypeSerDeParquetTest, NullableConversionFailuresBecomeNullOutsideStrictMode) { + auto expect_null = [](DataTypeSerDeSPtr serde, MutableColumnPtr column, + TestParquetDecodeSource* source, const ParquetDecodeContext& context) { + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + EXPECT_TRUE(serde->read_column_from_parquet(*column, *source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + }; + + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::min()}); + DataTypeDateV2 type; + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + DataTypeDateTimeV2 type(6); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS}); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({10}); + DataTypeDecimal32 type(1, 0); + expect_null(type.get_serde(), type.create_column(), &source, + {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 2, + .decimal_scale = 0}); + } +} + +TEST(DataTypeSerDeParquetTest, DictionaryConversionFailuresFollowDecodedIds) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0, 0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeInt8 type; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map[0], 1); + EXPECT_EQ(null_map[1], 1); + EXPECT_EQ(source.dictionary_decode_calls(), 1); +} + +TEST(DataTypeSerDeParquetTest, NonNullableDictionaryConversionFailureRemainsAnError) { + const int32_t overflow = 1000; + std::vector dictionary(sizeof(overflow)); + memcpy(dictionary.data(), &overflow, sizeof(overflow)); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(overflow), {0}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ParquetMaterializationState state; + DataTypeInt8 type; + auto column = type.create_column(); + + EXPECT_FALSE( + type.get_serde()->read_column_from_parquet(*column, source, context, 1, state).ok()); + EXPECT_EQ(column->size(), 0); +} + } // namespace } // namespace doris diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 55bed1de3e81d3..290ae994e253a3 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -208,6 +208,15 @@ std::vector encode_plain_byte_arrays(const std::vector& va return encoded; } +std::vector serialize_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, tparquet::CompressionCodec::type codec) { std::vector bytes; @@ -1129,6 +1138,156 @@ TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) EXPECT_TRUE(page_reader.next_page().is()); } +TEST(ParquetV2NativeDecoderTest, ColumnChunkRangeRejectsSignedOverflowAndBoundsLegacyPadding) { + tparquet::ColumnMetaData metadata; + metadata.__set_data_page_offset(-1); + metadata.__set_total_compressed_size(10); + ColumnChunkRange range; + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(95); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_data_page_offset(10); + metadata.__set_total_compressed_size(std::numeric_limits::max()); + EXPECT_TRUE( + compute_column_chunk_range(metadata, 100, false, &range).is()); + + metadata.__set_total_compressed_size(20); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, true, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 25); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlappingLocations) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(110); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(120); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 20)); + + second.__set_offset(140); + second.__set_first_row_index(0); + index.page_locations = {first, second}; + EXPECT_FALSE(validate_offset_index(index, range, 20)); + + second.__set_first_row_index(10); + index.page_locations = {first, second}; + EXPECT_TRUE(validate_offset_index(index, range, 20)); + + range = {.offset = std::numeric_limits::max(), .length = 2}; + EXPECT_FALSE(validate_offset_index(index, range, 20)); +} + +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingDataDecoder) { + tparquet::PageHeader index_header; + index_header.type = tparquet::PageType::INDEX_PAGE; + index_header.__set_compressed_page_size(3); + index_header.__set_uncompressed_page_size(3); + auto bytes = serialize_page(index_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + FieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); +} + +TEST(ParquetV2NativeDecoderTest, LegacyDataPageV2OverridesFalseCompressedFlag) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(1024); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + tparquet::ColumnMetaData metadata; + metadata.__set_codec(tparquet::CompressionCodec::SNAPPY); + EXPECT_TRUE(should_cache_decompressed(&header, metadata, false)); + EXPECT_FALSE(should_cache_decompressed(&header, metadata, true)); + + EXPECT_TRUE(parquet_reader_compat("parquet-cpp version 1.5.0").data_page_v2_always_compressed); + EXPECT_FALSE(parquet_reader_compat("parquet-cpp version 2.0.0").data_page_v2_always_compressed); + EXPECT_TRUE(parquet_reader_compat("parquet-mr version 1.2.8").parquet_816_padding); + EXPECT_FALSE(parquet_reader_compat("parquet-mr version 1.2.9").parquet_816_padding); +} + +TEST(ParquetV2NativeDecoderTest, NullableNumericOverflowIsNullOnlyOutsideStrictMode) { + const int32_t overflow = 1000; + auto decode = [&](bool strict, MutableColumnPtr* column, IColumn::Filter* null_map) { + std::unique_ptr decoder; + RETURN_IF_ERROR( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::PLAIN, decoder)); + decoder->set_type_length(sizeof(overflow)); + Slice slice(reinterpret_cast(&overflow), sizeof(overflow)); + RETURN_IF_ERROR(decoder->set_data(&slice)); + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + MutableColumnPtr column; + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + ASSERT_TRUE(decode(false, &column, &null_map).ok()); + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(null_map[0], 1); + + null_map.clear(); + null_map.resize_fill(1, 0); + EXPECT_FALSE(decode(true, &column, &null_map).ok()); + EXPECT_EQ(null_map[0], 0); +} + +TEST(ParquetV2NativeDecoderTest, FixedLengthStringsAppendAsOneContiguousSpan) { + ColumnString column; + const std::string values = "aaabbbccc"; + column.insert_many_fixed_length_data(values.data(), 3, 3); + ASSERT_EQ(column.size(), 3); + EXPECT_EQ(column.get_data_at(0).to_string_view(), "aaa"); + EXPECT_EQ(column.get_data_at(1).to_string_view(), "bbb"); + EXPECT_EQ(column.get_data_at(2).to_string_view(), "ccc"); +} + TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) { ColumnChunkReaderStatistics first_chunk; first_chunk.page_read_counter = 1; diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index b3d50f2c4154f8..79d9ebfd3886f5 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -177,6 +177,29 @@ format-specific checklist when reviewing Parquet or ORC. Arrow `ReadRecords`. Require profiles that distinguish page I/O, decompression, level decode, value decode, SerDe materialization, hybrid selection batches/ranges/NULL fallback, filtered-value skips, and page fragmentation. +- Validate every signed Column Chunk offset/length before converting it to `size_t`. The dictionary + offset is usable only when it is non-negative and precedes the data offset; the complete range + must fit the file. Apply the PARQUET-816 tail padding only to affected parquet-mr versions, cap it + at 100 bytes, and keep it inside the file. Scalar and levels-only COUNT readers share this helper. +- Treat OffsetIndex as one optional, all-or-nothing navigation structure. Require first row zero, + strictly increasing row ordinals and physical offsets, positive sizes, non-overlapping page + ranges, and containment in the owning Column Chunk. Discard a malformed index before selecting + the indexed reader. +- Page iteration skips `INDEX_PAGE` and unknown auxiliary pages before initializing a data decoder. + Dictionary pages retain their special first-page handling; a later dictionary page is corrupt. +- Derive writer workarounds once from `created_by` and pass them through scalar, nested, page-cache, + and COUNT paths. Pre-Arrow-3 parquet-cpp Data Page V2 payloads remain compressed despite the + historical `is_compressed=false` flag. +- Preserve nullable conversion semantics in direct native materialization. Numeric, DATE, + DATETIME, TIME, and DECIMAL failures insert a default nested value and mark the corresponding NULL + only in non-strict mode; strict or non-nullable reads return the error. Dictionary failures follow + the selected dictionary IDs to output rows. +- For cold small-file tests, separate footer I/O/Thrift parse from Arrow metadata adaptation. V2 may + retain the already-read serialized footer to avoid serializing the same Thrift object again; v1 + opens must not retain those bytes by default. +- Identical fixed-width POD values append with one bulk copy. FIXED_LEN_BYTE_ARRAY strings copy the + dense byte span once and synthesize offsets; validate this execution contract without flaky + wall-clock assertions. ## Parquet Multi-Level Filtering diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index c67ebeb1578adf..1220a432f63a11 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -400,7 +400,7 @@ that parsing step both feed the same level and value-decoder contracts. | Encoding family | Native responsibility | | --- | --- | -| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; sparse fixed-width ranges use bulk gather and sparse strings scan lengths once | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and sparse variable strings scan lengths once | | RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, decode and validate the complete ID batch, then gather selected IDs | | RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | | DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | @@ -427,6 +427,14 @@ exactly the size declared in the page header. For the UNCOMPRESSED codec, dictio uncompressed sizes must be equal. These rules turn malformed size metadata into corruption instead of a buffer overrun, silent truncation, or plausible shifted values. +Before a stream is created, scalar and levels-only readers call one signed, file-bounded Column +Chunk range validator. Writer compatibility derived from `created_by` may add at most 100 bytes of +file-bounded PARQUET-816 padding, or override the pre-Arrow-3 Data Page V2 compressed flag. Page +iteration ignores auxiliary `INDEX_PAGE`/unknown pages without consuming a logical data-page +ordinal. OffsetIndex is published only when its rows and non-overlapping physical page ranges are +strictly increasing and remain inside that validated Column Chunk; otherwise sequential traversal +preserves correctness. + ### 7.3 Direct Materialization and Scratch Reuse Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected @@ -457,6 +465,12 @@ checks prove the result is equivalent. In particular, legacy converted `TIMESTAM `TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct local/unspecified semantics; data decode and statistics pruning share this interpretation. +Direct conversion also preserves load-mode error semantics. A nullable target in non-strict mode +stores the nested default and marks the exact output row NULL for numeric, date/time, timestamp, and +decimal conversion failures. Strict mode and non-nullable targets return the error. Typed +dictionary materialization records failing dictionary entries once and propagates those failures +through decoded dictionary IDs, so dictionary and plain pages have identical behavior. + The typed dictionary is materialized through the logical SerDe once per decoder dictionary generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation @@ -625,7 +639,7 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | -| Footer metadata cache | Serialized footer bytes and immutable parsed native metadata | Stable file identity matching v1: path plus size and trustworthy modification/version information, with schema-affecting options in the parsed-object key | Avoid repeated footer I/O, Thrift parsing, and schema construction across scans | +| Footer metadata cache | Immutable parsed native metadata and, for a v2 cold miss, the serialized footer bytes already fetched from storage | Stable file identity matching v1: path plus size and trustworthy modification/version information, with schema-affecting options in the parsed-object key | Avoid repeated footer I/O, Thrift parsing, schema construction, and v2's former Thrift re-serialization before Arrow metadata adaptation | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | @@ -641,10 +655,11 @@ reusable entry. Parse failures, short files, encrypted/unsupported metadata, and option changes cannot populate or reuse a successful entry. V2 calls the same footer parser/cache and key builder as v1. On either a cache hit or miss, the -immutable native Thrift metadata is the owner. While Arrow planning remains, V2 serializes that -already cached Thrift object into Arrow's metadata parser, so opening the planner performs no second -footer read. The Arrow object is never the cache value and its lifetime does not enter the native -decoder. +immutable native Thrift metadata is the owner. On a v2 cold miss, the parser also retains the exact +serialized footer bytes already in memory, and the lazy Arrow planner adapter parses those bytes +directly instead of serializing the new Thrift object again. V1 keeps retention disabled by default. +The Arrow adapter is cached at the footer-object lifecycle; it is never the cache value and its +lifetime does not enter the native decoder. ### Page Cache Parity with V1 From c49ea39a24a9e50c082fe74efd30fefa53904143 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 15:45:14 +0800 Subject: [PATCH 12/34] [fix](be) Validate Parquet timestamp and offset index metadata ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: Parquet INT96 conversion could overflow signed arithmetic or accept an invalid nanos-of-day, TIMESTAMPTZ millisecond scaling could overflow or materialize values outside years 0001-9999, and a uniformly shifted OffsetIndex could still pass monotonic range validation. Use a shared checked timestamp conversion for direct, dictionary, and decoded/statistics paths, preserve strict rollback and non-strict NULL propagation, and anchor the first OffsetIndex location to the owning data_page_offset. ### Release note None ### Check List (For Author) - Test: Unit Test - DataTypeSerDeParquetTest.* and ParquetV2NativeDecoderTest.* (60 tests) - Behavior changed: Yes (invalid Parquet timestamps are rejected or converted to NULL according to load strictness, and incoherent OffsetIndex metadata falls back to sequential traversal) - Does this need documentation: Yes (updated the review guide and Parquet scan design in #65674) --- .../data_type_datetimev2_serde.cpp | 133 +++++------ .../data_type_timestamptz_serde.cpp | 124 +++++++---- .../core/data_type_serde/parquet_timestamp.h | 90 ++++++++ .../parquet/parquet_file_context.cpp | 6 +- .../reader/native/column_chunk_reader.cpp | 7 +- .../reader/native/column_chunk_reader.h | 2 +- .../data_type_serde_parquet_test.cpp | 210 ++++++++++++++++++ .../format_v2/parquet/native_decoder_test.cpp | 24 +- docs/file-scanner-v2-code-review-guide.md | 11 +- docs/file-scanner-v2-parquet-scan-design.md | 8 +- 10 files changed, 489 insertions(+), 126 deletions(-) create mode 100644 be/src/core/data_type_serde/parquet_timestamp.h diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index f5f1e83909d941..677bdc0a080afb 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -22,7 +22,6 @@ #include // IWYU pragma: keep #include -#include #include "common/config.h" #include "common/status.h" @@ -33,6 +32,7 @@ #include "core/data_type_serde/arrow_validation.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/types.h" #include "core/value/vdatetime_value.h" #include "exprs/function/cast/cast_to_datetimev2_impl.hpp" @@ -51,22 +51,6 @@ static const int64_t micro_to_nano_second = 1000; namespace { -#pragma pack(1) -struct DecodedInt96Timestamp { - int64_t nanos_of_day; - int32_t julian_day; - - int64_t to_timestamp_micros() const { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_IN_DAY = 86400000000; - static constexpr int64_t NANOS_PER_MICROSECOND = 1000; - return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + - nanos_of_day / NANOS_PER_MICROSECOND; - } -}; -#pragma pack() -static_assert(sizeof(DecodedInt96Timestamp) == 12); - Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; @@ -108,9 +92,9 @@ Status append_datetimev2_from_epoch_micros(ColumnDateTimeV2::Container& data, return Status::OK(); } -void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, - int64_t timestamp_micros, - const cctz::time_zone& timezone) { +Status append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, + int64_t timestamp_micros, + const cctz::time_zone& timezone) { static constexpr int64_t MICROS_PER_SECOND = 1000000; int64_t epoch_seconds = timestamp_micros / MICROS_PER_SECOND; @@ -123,35 +107,23 @@ void append_datetimev2_from_utc_epoch_micros(ColumnDateTimeV2::Container& data, DateV2Value datetime_value; datetime_value.from_unixtime(epoch_seconds, timezone); datetime_value.set_microsecond(static_cast(micros_of_second)); + if (!datetime_value.is_valid_date()) { + return Status::DataQualityError( + "Decoded DATETIMEV2 timestamp is outside the target timezone range: micros={}", + timestamp_micros); + } data.push_back(datetime_value); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; - } - return value; -} - -Status parquet_timestamp_micros(const ParquetDecodeContext& context, int64_t value, - int64_t* result) { - if (context.time_unit == ParquetTimeUnit::MILLIS) { - if (value > std::numeric_limits::max() / 1000 || - value < std::numeric_limits::min() / 1000) { - return Status::DataQualityError("Parquet timestamp overflows microseconds"); - } - *result = value * 1000; - return Status::OK(); - } - if (context.time_unit == ParquetTimeUnit::NANOS) { - *result = value / 1000; - return Status::OK(); + return ParquetTimeUnit::NANOS; } - *result = value; - return Status::OK(); + return ParquetTimeUnit::MICROS; } class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { @@ -168,19 +140,20 @@ class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { const auto& timezone = _context.timezone == nullptr ? utc_timezone : *_context.timezone; for (size_t row = 0; row < num_values; ++row) { int64_t timestamp_micros; + Status status; if (_context.physical_type == ParquetPhysicalType::INT96) { - DORIS_CHECK_EQ(value_width, sizeof(DecodedInt96Timestamp)); - timestamp_micros = unaligned_load( - values + row * sizeof(DecodedInt96Timestamp)) - .to_timestamp_micros(); - append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, timezone); - continue; + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); + } else { + DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); + DORIS_CHECK_EQ(value_width, sizeof(int64_t)); + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); } - DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); - DORIS_CHECK_EQ(value_width, sizeof(int64_t)); - auto status = parquet_timestamp_micros( - _context, unaligned_load(values + row * sizeof(int64_t)), - ×tamp_micros); if (!status.ok()) { if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { _data.emplace_back(); @@ -189,11 +162,11 @@ class DateTimeV2ParquetConsumer final : public ParquetFixedValueConsumer { _data.resize(old_size); return status; } - if (_context.timestamp_is_adjusted_to_utc) { - append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, timezone); - continue; - } - status = append_datetimev2_from_epoch_micros(_data, timestamp_micros); + status = _context.physical_type == ParquetPhysicalType::INT96 || + _context.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(_data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(_data, timestamp_micros); if (!status.ok()) { if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { _data.emplace_back(); @@ -645,7 +618,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( auto& data = assert_cast(column).get_data(); const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); static const auto utc_timezone = cctz::utc_time_zone(); const auto& timezone = view.timezone == nullptr ? utc_timezone : *view.timezone; for (int64_t row = 0; row < view.row_count; ++row) { @@ -653,8 +626,19 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - append_datetimev2_from_utc_epoch_micros(data, values[row].to_timestamp_micros(), - timezone); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -667,19 +651,22 @@ Status DataTypeDateTimeV2SerDe::read_column_from_decoded_values( data.push_back(DateV2Value()); continue; } - const int64_t timestamp_micros = decoded_timestamp_micros(view, values[row]); - if (view.timestamp_is_adjusted_to_utc) { - append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, timezone); - } else { - auto st = append_datetimev2_from_epoch_micros(data, timestamp_micros); - if (!st.ok()) { - if (decoded_column_view_can_null_on_conversion_failure(view)) { - decoded_column_view_insert_null_on_conversion_failure(column, view, row); - continue; - } - data.resize(old_size); - return st; + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = view.timestamp_is_adjusted_to_utc + ? append_datetimev2_from_utc_epoch_micros(data, timestamp_micros, + timezone) + : append_datetimev2_from_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; } + data.resize(old_size); + return status; } } return Status::OK(); diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index 3e2e67e472e886..4278a8f30f55ab 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -23,6 +23,7 @@ #include "core/data_type/primitive_type.h" #include "core/data_type_serde/decoded_column_view.h" #include "core/data_type_serde/parquet_decode_source.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "core/value/timestamptz_value.h" #include "exprs/function/cast/cast_parameters.h" #include "exprs/function/cast/cast_to_string.h" @@ -32,24 +33,8 @@ namespace doris { namespace { -#pragma pack(1) -struct DecodedInt96Timestamp { - int64_t nanos_of_day; - int32_t julian_day; - - int64_t to_timestamp_micros() const { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_IN_DAY = 86400000000; - static constexpr int64_t NANOS_PER_MICROSECOND = 1000; - return (julian_day - JULIAN_EPOCH_OFFSET_DAYS) * MICROS_IN_DAY + - nanos_of_day / NANOS_PER_MICROSECOND; - } -}; -#pragma pack() -static_assert(sizeof(DecodedInt96Timestamp) == 12); - -void append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& data, - int64_t timestamp_micros) { +Status append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& data, + int64_t timestamp_micros) { static constexpr int64_t MICROS_PER_SECOND = 1000000; static const auto UTC = cctz::utc_time_zone(); @@ -63,43 +48,62 @@ void append_timestamptz_from_utc_epoch_micros(ColumnTimeStampTz::Container& data TimestampTzValue timestamp_tz; timestamp_tz.from_unixtime(epoch_seconds, UTC); timestamp_tz.set_microsecond(static_cast(micros_of_second)); + if (!timestamp_tz.is_valid_date()) { + return Status::DataQualityError( + "Decoded TIMESTAMPTZ is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } data.push_back(timestamp_tz); + return Status::OK(); } -int64_t decoded_timestamp_micros(const DecodedColumnView& view, int64_t value) { +ParquetTimeUnit decoded_parquet_time_unit(const DecodedColumnView& view) { if (view.time_unit == DecodedTimeUnit::MILLIS) { - return value * 1000; + return ParquetTimeUnit::MILLIS; } if (view.time_unit == DecodedTimeUnit::NANOS) { - return value / 1000; + return ParquetTimeUnit::NANOS; } - return value; + return ParquetTimeUnit::MICROS; } class TimestampTzParquetConsumer final : public ParquetFixedValueConsumer { public: - TimestampTzParquetConsumer(IColumn& column, const ParquetDecodeContext& context) - : _data(assert_cast(column).get_data()), _context(context) {} + TimestampTzParquetConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state = nullptr) + : _data(assert_cast(column).get_data()), + _context(context), + _state(state) {} Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _data.size(); for (size_t row = 0; row < num_values; ++row) { int64_t timestamp_micros; + Status status; if (_context.physical_type == ParquetPhysicalType::INT96) { - DORIS_CHECK_EQ(value_width, sizeof(DecodedInt96Timestamp)); - timestamp_micros = unaligned_load( - values + row * sizeof(DecodedInt96Timestamp)) - .to_timestamp_micros(); + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + status = parquet_int96_timestamp_micros( + unaligned_load(values + + row * sizeof(ParquetInt96Timestamp)), + ×tamp_micros); } else { DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); DORIS_CHECK_EQ(value_width, sizeof(int64_t)); - timestamp_micros = unaligned_load(values + row * sizeof(int64_t)); - if (_context.time_unit == ParquetTimeUnit::MILLIS) { - timestamp_micros *= 1000; - } else if (_context.time_unit == ParquetTimeUnit::NANOS) { - timestamp_micros /= 1000; + status = parquet_timestamp_micros( + _context.time_unit, unaligned_load(values + row * sizeof(int64_t)), + ×tamp_micros); + } + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); + } + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; } + _data.resize(old_size); + return status; } - append_timestamptz_from_utc_epoch_micros(_data, timestamp_micros); } return Status::OK(); } @@ -107,6 +111,7 @@ class TimestampTzParquetConsumer final : public ParquetFixedValueConsumer { private: ColumnTimeStampTz::Container& _data; const ParquetDecodeContext& _context; + ParquetMaterializationState* _state; }; class RejectTimestampTzBinaryConsumer final : public ParquetBinaryValueConsumer { @@ -350,14 +355,27 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( } auto& data = assert_cast(column).get_data(); + const auto old_size = data.size(); if (view.value_kind == DecodedValueKind::INT96) { - const auto* values = reinterpret_cast(view.values); + const auto* values = reinterpret_cast(view.values); for (int64_t row = 0; row < view.row_count; ++row) { if (decoded_column_view_row_is_null(view, row)) { data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, values[row].to_timestamp_micros()); + int64_t timestamp_micros; + auto status = parquet_int96_timestamp_micros(values[row], ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -368,7 +386,20 @@ Status DataTypeTimeStampTzSerDe::read_column_from_decoded_values( data.push_back(TimestampTzValue()); continue; } - append_timestamptz_from_utc_epoch_micros(data, decoded_timestamp_micros(view, values[row])); + int64_t timestamp_micros; + auto status = parquet_timestamp_micros(decoded_parquet_time_unit(view), values[row], + ×tamp_micros); + if (status.ok()) { + status = append_timestamptz_from_utc_epoch_micros(data, timestamp_micros); + } + if (!status.ok()) { + if (decoded_column_view_can_null_on_conversion_failure(view)) { + decoded_column_view_insert_null_on_conversion_failure(column, view, row); + continue; + } + data.resize(old_size); + return status; + } } return Status::OK(); } @@ -387,20 +418,35 @@ Status DataTypeTimeStampTzSerDe::read_column_from_parquet( context.physical_type != ParquetPhysicalType::INT96) { return Status::NotSupported("TIMESTAMPTZ expects Parquet INT64 or INT96"); } - TimestampTzParquetConsumer consumer(column, context); + TimestampTzParquetConsumer consumer(column, context, &state); if (context.encoding != ParquetValueEncoding::DICTIONARY) { return source.decode_fixed_values(num_values, consumer); } if (state.dictionary_generation != source.dictionary_generation()) { state.typed_dictionary = column.clone_empty(); - RETURN_IF_ERROR(read_parquet_dictionary(*state.typed_dictionary, source, context)); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + TimestampTzParquetConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectTimestampTzBinaryConsumer binary_consumer; + const Status dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < num_values; ++row) { + if (!state.dictionary_conversion_failures.empty() && + state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + state.mark_conversion_failure(old_size + row); + } + } + } return Status::OK(); } diff --git a/be/src/core/data_type_serde/parquet_timestamp.h b/be/src/core/data_type_serde/parquet_timestamp.h new file mode 100644 index 00000000000000..87ff18d28ce331 --- /dev/null +++ b/be/src/core/data_type_serde/parquet_timestamp.h @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "common/status.h" +#include "core/data_type_serde/parquet_decode_source.h" + +namespace doris { + +#pragma pack(1) +struct ParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(ParquetInt96Timestamp) == 12); + +inline constexpr int64_t MIN_DORIS_TIMESTAMP_MICROS = -62135596800000000LL; +inline constexpr int64_t MAX_DORIS_TIMESTAMP_MICROS = 253402300799999999LL; + +inline Status validate_parquet_timestamp_micros(int64_t timestamp_micros) { + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet timestamp is outside the Doris 0001-9999 range: micros={}", + timestamp_micros); + } + return Status::OK(); +} + +inline Status parquet_timestamp_micros(ParquetTimeUnit unit, int64_t value, int64_t* result) { + if (unit == ParquetTimeUnit::MILLIS) { + // Validate in the source unit before scaling; signed overflow could otherwise turn an + // invalid file value into an in-range timestamp that survives later range checks. + if (value > std::numeric_limits::max() / 1000 || + value < std::numeric_limits::min() / 1000) { + return Status::DataQualityError("Parquet timestamp overflows microseconds"); + } + *result = value * 1000; + } else if (unit == ParquetTimeUnit::NANOS) { + *result = value / 1000; + } else { + *result = value; + } + return validate_parquet_timestamp_micros(*result); +} + +inline Status parquet_int96_timestamp_micros(const ParquetInt96Timestamp& value, int64_t* result) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_IN_DAY = 86400000000LL; + static constexpr int64_t NANOS_IN_DAY = 86400000000000LL; + static constexpr int64_t NANOS_PER_MICROSECOND = 1000; + + // INT96 nanos is a time-of-day, not a signed duration. Validate it before widened day + // arithmetic so corrupt input cannot wrap into a plausible Doris timestamp. + if (value.nanos_of_day < 0 || value.nanos_of_day >= NANOS_IN_DAY) { + return Status::DataQualityError("Invalid Parquet INT96 nanos-of-day: {}", + value.nanos_of_day); + } + const __int128 days = static_cast(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + const __int128 timestamp_micros = + days * MICROS_IN_DAY + value.nanos_of_day / NANOS_PER_MICROSECOND; + if (timestamp_micros < MIN_DORIS_TIMESTAMP_MICROS || + timestamp_micros > MAX_DORIS_TIMESTAMP_MICROS) { + return Status::DataQualityError( + "Parquet INT96 timestamp is outside the Doris 0001-9999 range"); + } + *result = static_cast(timestamp_micros); + return Status::OK(); +} + +} // namespace doris diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index d7bbc39245d4da..69e035b43bc9ff 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -691,8 +691,10 @@ Status ParquetFileContext::load_native_offset_indexes( RETURN_IF_ERROR(native::compute_column_chunk_range( native_row_group.columns[leaf_column_id].meta_data, native_file->size(), compat.parquet_816_padding, &chunk_range)); - if (!native::validate_offset_index(native_index, chunk_range, - native_row_group.num_rows)) { + if (!native::validate_offset_index( + native_index, chunk_range, + native_row_group.columns[leaf_column_id].meta_data.data_page_offset, + native_row_group.num_rows)) { // OffsetIndex is optional. Reject the complete index instead of letting one bad // location redirect an indexed reader outside its owning column chunk. continue; diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 7b29bfcad70272..9537aa03d4dda3 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -97,12 +97,15 @@ Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size } bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, - int64_t row_count) { - if (index.page_locations.empty() || row_count < 0 || + int64_t data_page_offset, int64_t row_count) { + if (index.page_locations.empty() || data_page_offset < 0 || row_count < 0 || index.page_locations.front().first_row_index != 0 || + index.page_locations.front().offset != data_page_offset || chunk_range.length > std::numeric_limits::max() - chunk_range.offset) { return false; } + // Row indexes alone cannot detect a uniformly shifted OffsetIndex. Anchor its first location + // to the owning metadata so page-to-row mapping cannot silently move by one physical page. const uint64_t chunk_begin = chunk_range.offset; const uint64_t chunk_end = chunk_begin + chunk_range.length; uint64_t previous_end = chunk_begin; diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index ec27e5e18c61e6..9b7c1f27571ab8 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -66,7 +66,7 @@ ParquetReaderCompat parquet_reader_compat(const std::string& created_by); Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size_t file_size, bool parquet_816_padding, ColumnChunkRange* range); bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, - int64_t row_count); + int64_t data_page_offset, int64_t row_count); struct ColumnChunkReaderStatistics { int64_t decompress_time = 0; diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp index 80a72670300c37..1dec32bf8acebc 100644 --- a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -39,6 +39,17 @@ namespace doris { namespace { +#pragma pack(1) +struct TestParquetInt96Timestamp { + int64_t nanos_of_day; + int32_t julian_day; +}; +#pragma pack() +static_assert(sizeof(TestParquetInt96Timestamp) == 12); + +constexpr int32_t JULIAN_UNIX_EPOCH = 2440588; +constexpr int64_t NANOS_PER_DAY = 86400000000000LL; + class TestParquetDecodeSource final : public ParquetDecodeSource { public: template @@ -299,6 +310,205 @@ TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { EXPECT_EQ(data[1].microsecond(), 1); } +TEST(DataTypeSerDeParquetTest, ValidatesInt96BeforeDateTimeMaterialization) { + { + TestParquetDecodeSource source; + source.set_fixed_values( + {{0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY - 1, JULIAN_UNIX_EPOCH}}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 23:59:59.999999"); + } + + const std::vector invalid_values { + {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}, + {-1, JULIAN_UNIX_EPOCH}, + {0, std::numeric_limits::max()}}; + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + ParquetMaterializationState state; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values(invalid_values); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 3, state) + .ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, Int96DictionaryFailuresFollowDecodedIds) { + const std::vector dictionary_values { + {0, JULIAN_UNIX_EPOCH}, {NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + std::vector dictionary(sizeof(TestParquetInt96Timestamp) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(TestParquetInt96Timestamp), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT96, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(type.to_string(*column, 1), "1970-01-01 00:00:00.000000"); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzChecksUnitOverflowAndTargetRange) { + constexpr int64_t MIN_TIMESTAMP_MICROS = -62135596800000000LL; + constexpr int64_t MAX_TIMESTAMP_MICROS = 253402300799999999LL; + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + + { + TestParquetDecodeSource source; + source.set_fixed_values({MIN_TIMESTAMP_MICROS, MAX_TIMESTAMP_MICROS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].year(), 1); + EXPECT_EQ(data[1].year(), 9999); + EXPECT_EQ(data[1].microsecond(), 999999); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max()}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + ParquetMaterializationState state; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({std::numeric_limits::max(), YEAR_10000_MILLIS}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + EXPECT_EQ(column->size(), 2); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDecodedValuesUseTheSameBounds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector values {std::numeric_limits::max(), YEAR_10000_MILLIS, 0}; + NullMap conversion_failures(3, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT64, + .time_unit = DecodedTimeUnit::MILLIS, + .row_count = 3, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(conversion_failures, NullMap({1, 1, 0})); + EXPECT_EQ(assert_cast(*column).get_data()[2].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, TimestampTzDictionaryFailuresFollowDecodedIds) { + constexpr int64_t YEAR_10000_MILLIS = 253402300800000LL; + const std::vector dictionary_values {0, YEAR_10000_MILLIS}; + std::vector dictionary(sizeof(int64_t) * dictionary_values.size()); + memcpy(dictionary.data(), dictionary_values.data(), dictionary.size()); + TestParquetDecodeSource source; + source.set_dictionary(std::move(dictionary), sizeof(int64_t), {1, 0, 1}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .encoding = ParquetValueEncoding::DICTIONARY, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MILLIS, + .timestamp_is_adjusted_to_utc = true}; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + DataTypeTimeStampTz type(6); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + EXPECT_EQ(column->size(), 3); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + EXPECT_EQ(assert_cast(*column).get_data()[1].year(), 1970); +} + +TEST(DataTypeSerDeParquetTest, Int96DecodedValuesRejectInvalidNanos) { + const std::vector values {{NANOS_PER_DAY, JULIAN_UNIX_EPOCH}}; + NullMap conversion_failures(1, 0); + DecodedColumnView view {.value_kind = DecodedValueKind::INT96, + .row_count = 1, + .values = reinterpret_cast(values.data()), + .enable_strict_mode = false, + .conversion_failure_null_map = &conversion_failures}; + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde()->read_column_from_decoded_values(*column, view).ok()); + EXPECT_EQ(column->size(), 1); + EXPECT_EQ(conversion_failures, NullMap({1})); +} + TEST(DataTypeSerDeParquetTest, MaterializesFixedBinaryAsVarbinaryDirectly) { TestParquetDecodeSource source; source.set_fixed_bytes({0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 290ae994e253a3..5fa45d77b223d3 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -1173,19 +1173,35 @@ TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlapp second.__set_compressed_page_size(20); second.__set_first_row_index(10); index.page_locations = {first, second}; - EXPECT_FALSE(validate_offset_index(index, range, 20)); + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); second.__set_offset(140); second.__set_first_row_index(0); index.page_locations = {first, second}; - EXPECT_FALSE(validate_offset_index(index, range, 20)); + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); second.__set_first_row_index(10); index.page_locations = {first, second}; - EXPECT_TRUE(validate_offset_index(index, range, 20)); + EXPECT_TRUE(validate_offset_index(index, range, 110, 20)); range = {.offset = std::numeric_limits::max(), .length = 2}; - EXPECT_FALSE(validate_offset_index(index, range, 20)); + EXPECT_FALSE(validate_offset_index(index, range, 110, 20)); +} + +TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsShiftedFirstDataPage) { + ColumnChunkRange range {.offset = 100, .length = 100}; + tparquet::OffsetIndex index; + tparquet::PageLocation first; + first.__set_offset(120); + first.__set_compressed_page_size(20); + first.__set_first_row_index(0); + tparquet::PageLocation second; + second.__set_offset(140); + second.__set_compressed_page_size(20); + second.__set_first_row_index(10); + index.page_locations = {first, second}; + + EXPECT_FALSE(validate_offset_index(index, range, 100, 20)); } TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingDataDecoder) { diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index 79d9ebfd3886f5..e86808bfda7e61 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -129,6 +129,10 @@ format-specific checklist when reviewing Parquet or ORC. - Treat legacy Parquet `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` converted types as UTC-adjusted. Do not give them the local/unspecified semantics of an unannotated INT64 timestamp; data decode, statistics conversion, and min/max pruning must use the same timezone rule. +- Route plain, dictionary, and decoded timestamp inputs through one checked conversion contract. + Validate INT96 nanos-of-day before widened Julian-day arithmetic, reject unit scaling overflow, + and enforce Doris year 0001-9999 before materialization. Conversion failures must follow the same + strict/non-strict and dictionary-ID propagation rules as other direct types. - Verify schema-change routing separately from physical decode. Integer, FLOAT-to-DOUBLE, decimal, and string-family changes should use the direct target-SerDe path. Other supported logical casts may use one persistent generic `ColumnTypeConverter` source column; its value/null-map sizes must @@ -182,9 +186,10 @@ format-specific checklist when reviewing Parquet or ORC. must fit the file. Apply the PARQUET-816 tail padding only to affected parquet-mr versions, cap it at 100 bytes, and keep it inside the file. Scalar and levels-only COUNT readers share this helper. - Treat OffsetIndex as one optional, all-or-nothing navigation structure. Require first row zero, - strictly increasing row ordinals and physical offsets, positive sizes, non-overlapping page - ranges, and containment in the owning Column Chunk. Discard a malformed index before selecting - the indexed reader. + the first physical location to equal the owning ColumnMetaData `data_page_offset`, strictly + increasing row ordinals and physical offsets, positive sizes, non-overlapping page ranges, and + containment in the owning Column Chunk. Discard a malformed index before selecting the indexed + reader. - Page iteration skips `INDEX_PAGE` and unknown auxiliary pages before initializing a data decoder. Dictionary pages retain their special first-page handling; a later dictionary page is corrupt. - Derive writer workarounds once from `created_by` and pass them through scalar, nested, page-cache, diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 1220a432f63a11..5970381eab73d6 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -432,8 +432,9 @@ Chunk range validator. Writer compatibility derived from `created_by` may add at file-bounded PARQUET-816 padding, or override the pre-Arrow-3 Data Page V2 compressed flag. Page iteration ignores auxiliary `INDEX_PAGE`/unknown pages without consuming a logical data-page ordinal. OffsetIndex is published only when its rows and non-overlapping physical page ranges are -strictly increasing and remain inside that validated Column Chunk; otherwise sequential traversal -preserves correctness. +strictly increasing, its first location equals the owning `data_page_offset`, and every location +remains inside that validated Column Chunk. The metadata anchor rejects uniformly shifted indexes +that row monotonicity alone cannot detect; otherwise sequential traversal preserves correctness. ### 7.3 Direct Materialization and Scratch Reuse @@ -464,6 +465,9 @@ the same semantic checks as the general conversion path. A fast path is enabled checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct local/unspecified semantics; data decode and statistics pruning share this interpretation. +Timestamp conversion validates millisecond scaling before multiplication, validates INT96 +nanos-of-day before widened Julian-day arithmetic, and rejects values outside Doris years +0001-9999. Plain, dictionary, and decoded-value inputs share these bounds. Direct conversion also preserves load-mode error semantics. A nullable target in non-strict mode stores the nested default and marks the exact output row NULL for numeric, date/time, timestamp, and From 2da6d7e9b9770d433af0323ae816c0e29e6ea565 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 17 Jul 2026 19:06:45 +0800 Subject: [PATCH 13/34] [fix](file scanner) Address parquet v2 review feedback --- be/src/core/column/column_string.h | 50 +++ .../data_type_decimal_serde.cpp | 125 ++++-- .../data_type_number_serde.cpp | 34 ++ .../data_type_string_serde.cpp | 17 + .../data_type_serde/data_type_time_serde.cpp | 12 +- .../data_type_varbinary_serde.cpp | 10 + .../data_type_serde/parquet_decode_source.h | 17 + be/src/format/column_type_convert.cpp | 7 +- be/src/format/parquet/parquet_thrift_util.h | 12 +- .../format/parquet/vparquet_file_metadata.cpp | 56 +-- .../format/parquet/vparquet_file_metadata.h | 17 +- be/src/format/parquet/vparquet_reader.cpp | 3 +- .../parquet/parquet_file_context.cpp | 367 ++++++++++++++++-- .../format_v2/parquet/parquet_file_context.h | 52 ++- be/src/format_v2/parquet/parquet_profile.cpp | 13 + be/src/format_v2/parquet/parquet_profile.h | 8 + be/src/format_v2/parquet/parquet_reader.cpp | 6 +- be/src/format_v2/parquet/parquet_scan.cpp | 329 +++++++++++----- be/src/format_v2/parquet/parquet_scan.h | 26 +- .../format_v2/parquet/parquet_statistics.cpp | 240 ++++++++++++ be/src/format_v2/parquet/parquet_statistics.h | 19 + .../parquet/reader/count_column_reader.cpp | 4 +- .../parquet/reader/count_column_reader.h | 6 +- .../native/byte_array_plain_decoder.cpp | 57 ++- .../reader/native/byte_array_plain_decoder.h | 14 +- .../reader/native/column_chunk_reader.cpp | 17 +- .../parquet/reader/native/level_decoder.cpp | 31 +- .../parquet/reader/native/level_decoder.h | 12 + .../parquet/reader/native_column_reader.cpp | 24 +- .../parquet/reader/native_column_reader.h | 18 +- be/src/io/fs/file_meta_cache.cpp | 11 - be/src/io/fs/file_meta_cache.h | 4 - .../data_type_serde_parquet_test.cpp | 139 +++++++ .../file_reader/file_meta_cache_test.cpp | 7 - .../format/parquet/parquet_thrift_test.cpp | 18 - .../format_v2/parquet/native_decoder_test.cpp | 163 ++++++++ .../format_v2/parquet/parquet_scan_test.cpp | 84 +++- docs/file-scanner-v2-code-review-guide.md | 37 +- docs/file-scanner-v2-parquet-scan-design.md | 94 +++-- 39 files changed, 1780 insertions(+), 380 deletions(-) diff --git a/be/src/core/column/column_string.h b/be/src/core/column/column_string.h index b864a20debfe18..616d6ef9df39e1 100644 --- a/be/src/core/column/column_string.h +++ b/be/src/core/column/column_string.h @@ -277,6 +277,56 @@ class ColumnStr final : public COWHelper> { sanity_check_simple(); } + template + void insert_many_parquet_plain_byte_arrays(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num, + const std::vector& value_spans) { + if (UNLIKELY(num == 0)) { + return; + } + const size_t old_chars_size = chars.size(); + const size_t bytes = value_offsets[num]; + check_chars_length(old_chars_size + bytes, offsets.size() + num); + chars.resize(old_chars_size + bytes); + + // Source payloads may be contiguous for decoder-owned buffers even though PLAIN normally + // separates them with length prefixes. Coalesce whenever the published layout permits it; + // the fallback remains one direct source-to-column copy without a StringRef staging array. + size_t covered_values = 0; + for (const auto& span : value_spans) { + DORIS_CHECK_EQ(span.first, covered_values); + DORIS_CHECK_LE(span.first + span.count, num); + size_t run_first = span.first; + const size_t span_end = span.first + span.count; + while (run_first < span_end) { + size_t run_end = run_first + 1; + while (run_end < span_end && + payload_offsets[run_end] == payload_offsets[run_first] + + value_offsets[run_end] - + value_offsets[run_first]) { + ++run_end; + } + const size_t run_bytes = value_offsets[run_end] - value_offsets[run_first]; + if (run_bytes != 0) { + memcpy(chars.data() + old_chars_size + value_offsets[run_first], + encoded_data + payload_offsets[run_first], run_bytes); + } + run_first = run_end; + } + covered_values = span_end; + } + DORIS_CHECK_EQ(covered_values, num); + + const size_t old_rows = offsets.size(); + const auto tail_offset = offsets.back(); + offsets.resize(old_rows + num); + for (size_t row = 0; row < num; ++row) { + offsets[old_rows + row] = tail_offset + value_offsets[row + 1]; + } + sanity_check_simple(); + } + // Insert `num` string entries with real length information but no actual // character data. The `lengths` array provides the byte length of each // string. Offsets are built with correct cumulative sizes so that diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index 7578661c16f8dd..5648406a974628 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -189,24 +190,65 @@ Status read_decimal_decoded_values(IColumn& column, const DecodedColumnView& vie } template -Status scale_parquet_decimal(typename PrimitiveTypeTraits::CppType::NativeType value, - int32_t source_scale, int32_t target_scale, - typename PrimitiveTypeTraits::CppType::NativeType* result) { - using NativeType = typename PrimitiveTypeTraits::CppType::NativeType; - DORIS_CHECK(result != nullptr); - if (source_scale == target_scale) { - *result = value; - return Status::OK(); +wide::Int256 parquet_decimal_limit(UInt32 precision) { + if constexpr (T == TYPE_DECIMALV2) { + return wide::Int256(DataTypeDecimal::get_max_digits_number(precision)); + } else { + return wide::Int256(max_decimal_value(precision).value); } +} + +template +Status scale_parquet_decimal(wide::Int256 value, int32_t source_scale, int32_t target_scale, + UInt32 target_precision, wide::Int256* result) { + DORIS_CHECK(result != nullptr); + const auto limit = parquet_decimal_limit(target_precision); if (source_scale > target_scale) { - *result = value / decimal_scale_multiplier(source_scale - target_scale); - return Status::OK(); + const int64_t scale_delta = static_cast(source_scale) - target_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // The precision bound above guarantees that narrowing the positive scale delta keeps + // the multiplier lookup in range. + const auto divisor = + decimal_scale_multiplier(static_cast(scale_delta)); + // Scale-down must be exact. Truncating before the target precision check silently + // changes source values and makes plain and dictionary decoding disagree with casts. + if (value % divisor != 0) { + return Status::DataQualityError( + "Parquet decimal loses precision while scaling from {} to {}", source_scale, + target_scale); + } + value /= divisor; + } + } else if (source_scale < target_scale) { + const int64_t scale_delta = static_cast(target_scale) - source_scale; + if (scale_delta > BeConsts::MAX_DECIMAL256_PRECISION) { + if (value != 0) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + } else { + // Keep the same checked narrowing invariant for scale-up as for exact scale-down. + const auto multiplier = + decimal_scale_multiplier(static_cast(scale_delta)); + if (value > limit / multiplier || value < -limit / multiplier) { + return Status::DataQualityError( + "Parquet decimal overflows while scaling from {} to {}", source_scale, + target_scale); + } + value *= multiplier; + } } - const auto multiplier = decimal_scale_multiplier(target_scale - source_scale); - if (common::mul_overflow(value, multiplier, *result)) { - return Status::DataQualityError("Parquet decimal overflows while scaling from {} to {}", - source_scale, target_scale); + if (value < -limit || value > limit) { + return Status::DataQualityError("Parquet decimal value is out of range"); } + *result = value; return Status::OK(); } @@ -275,14 +317,48 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, return Status::OK(); } + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + const size_t old_size = _data.size(); + _data.resize(old_size + num_values); + for (size_t row = 0; row < num_values; ++row) { + const size_t length = value_offsets[row + 1] - value_offsets[row]; + auto status = append_binary_value( + reinterpret_cast(encoded_data + payload_offsets[row]), length, + old_size + row); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = FieldType(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + private: template Status append_integers(const uint8_t* values, size_t num_values) { const size_t old_size = _data.size(); _data.resize(old_size + num_values); + constexpr int32_t SOURCE_DIGITS = std::numeric_limits::digits10 + 1; + if (_context.decimal_scale == _target_scale && + SOURCE_DIGITS <= static_cast(_target_precision)) { + // The complete physical domain fits the target at the same scale, so narrowing cannot + // precede a failure check and the hot same-scale path needs no wide arithmetic. + for (size_t row = 0; row < num_values; ++row) { + const auto source_value = + unaligned_load(values + row * sizeof(SourceType)); + _data[old_size + row] = FieldType {NativeType(source_value)}; + } + return Status::OK(); + } for (size_t row = 0; row < num_values; ++row) { const auto source_value = unaligned_load(values + row * sizeof(SourceType)); - auto status = append_native_value(NativeType(source_value), old_size + row); + auto status = append_wide_value(wide::Int256(source_value), old_size + row); if (!status.ok()) { if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { _data[old_size + row] = FieldType(); @@ -296,22 +372,21 @@ class DecimalParquetConsumer final : public ParquetFixedValueConsumer, } Status append_binary_value(const uint8_t* value, size_t length, size_t output_row) { - if (UNLIKELY(length > sizeof(NativeType))) { + if (UNLIKELY(length > sizeof(wide::Int256))) { return Status::DataQualityError("Parquet decimal binary value is too wide: {}", length); } - return append_native_value( - decode_big_endian_signed_integer(value, cast_set(length)), + return append_wide_value( + decode_big_endian_signed_integer(value, cast_set(length)), output_row); } - Status append_native_value(NativeType value, size_t output_row) { - NativeType scaled_value; + Status append_wide_value(wide::Int256 value, size_t output_row) { + wide::Int256 scaled_value; RETURN_IF_ERROR(scale_parquet_decimal(value, _context.decimal_scale, _target_scale, - &scaled_value)); - if (!decoded_decimal_value_fits(scaled_value, _target_precision)) { - return Status::DataQualityError("Parquet decimal value is out of range"); - } - _data[output_row] = FieldType {scaled_value}; + _target_precision, &scaled_value)); + // Narrow only after scaling and target-precision validation. In particular, an INT64 or a + // sign-extended binary value must never wrap through Decimal32 before it can be rejected. + _data[output_row] = FieldType {static_cast(scaled_value)}; return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index f3fcc4978ad4e0..6ef52bedd42b10 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -98,6 +98,22 @@ bool decoded_number_value_fits(SourceType value) { } } +template +constexpr bool parquet_number_conversion_always_fits() { + if constexpr (std::is_floating_point_v) { + return true; + } else if constexpr (!std::is_integral_v || !std::is_integral_v || + std::is_same_v) { + return false; + } else if constexpr (std::is_signed_v == std::is_signed_v) { + return sizeof(DorisCppType) >= sizeof(SourceType); + } else if constexpr (std::is_signed_v) { + return sizeof(DorisCppType) > sizeof(SourceType); + } else { + return false; + } +} + template Status read_number_decoded_values(IColumn& column, const DecodedColumnView& view) { if (view.values == nullptr && decoded_column_view_has_non_null_value(view)) { @@ -212,6 +228,15 @@ Status append_parquet_number(PaddedPODArray& data, const uint8_t* memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); return Status::OK(); } + if constexpr (parquet_number_conversion_always_fits()) { + // A widening conversion cannot fail, so keeping range checks in the row loop only blocks + // auto-vectorization. Input can be unaligned at a Parquet page boundary; load explicitly. + for (size_t row = 0; row < num_values; ++row) { + data[old_size + row] = static_cast( + unaligned_load(values + row * sizeof(SourceType))); + } + return Status::OK(); + } for (size_t row = 0; row < num_values; ++row) { const auto value = unaligned_load(values + row * sizeof(SourceType)); if (!decoded_number_value_fits(value)) { @@ -234,6 +259,15 @@ Status append_parquet_logical_integers(PaddedPODArray& data, const size_t num_values, ParquetMaterializationState* state) { const size_t old_size = data.size(); data.resize(old_size + num_values); + if constexpr (parquet_number_conversion_always_fits()) { + for (size_t row = 0; row < num_values; ++row) { + const auto physical_value = + unaligned_load(values + row * sizeof(SourceType)); + data[old_size + row] = + static_cast(static_cast(physical_value)); + } + return Status::OK(); + } for (size_t row = 0; row < num_values; ++row) { const auto physical_value = unaligned_load(values + row * sizeof(SourceType)); const auto logical_value = static_cast(physical_value); diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index b9f170cb95b8ff..01130b0ddf726e 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -99,6 +99,23 @@ class StringParquetConsumer final : public ParquetFixedValueConsumer, return Status::OK(); } + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + if constexpr (requires(ColumnType& column) { + column.insert_many_parquet_plain_byte_arrays( + encoded_data, payload_offsets, value_offsets, num_values, + value_spans); + }) { + _column.insert_many_parquet_plain_byte_arrays(encoded_data, payload_offsets, + value_offsets, num_values, value_spans); + return Status::OK(); + } + return ParquetBinaryValueConsumer::consume_plain_byte_array( + encoded_data, payload_offsets, value_offsets, num_values, value_spans); + } + private: ColumnType& _column; }; diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index 793be706b4aaf5..7fb505085c88c6 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -93,15 +93,9 @@ class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { micros /= 1000; } } - const bool negative = micros < 0; - const uint64_t abs_micros = negative ? uint64_t(-(micros + 1)) + 1 : uint64_t(micros); - _data[old_size + row] = - TimeValue::make_time(abs_micros / TimeValue::ONE_HOUR_MICROSECONDS, - (abs_micros % TimeValue::ONE_HOUR_MICROSECONDS) / - TimeValue::ONE_MINUTE_MICROSECONDS, - (abs_micros % TimeValue::ONE_MINUTE_MICROSECONDS) / - TimeValue::ONE_SECOND_MICROSECONDS, - abs_micros % TimeValue::ONE_SECOND_MICROSECONDS, negative); + // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields + // and immediately recombining them is an identity operation with several divisions. + _data[old_size + row] = static_cast(micros); } return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp index e0880caaea3b48..9bdd5abfcc2bfe 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp @@ -44,6 +44,16 @@ class VarbinaryParquetConsumer final : public ParquetFixedValueConsumer, return Status::OK(); } + Status consume_plain_byte_array(const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector&) override { + for (size_t row = 0; row < num_values; ++row) { + _column.insert_data(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return Status::OK(); + } + private: ColumnVarbinary& _column; }; diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index ec6e9e6941d876..b9a3a8c731bb6e 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -121,6 +121,23 @@ class ParquetBinaryValueConsumer { public: virtual ~ParquetBinaryValueConsumer() = default; virtual Status consume(const StringRef* values, size_t num_values) = 0; + + // PLAIN BYTE_ARRAY decoders already have to parse every length prefix. Publish the parsed + // source and destination offsets so string columns do not rebuild an equally large StringRef + // array and rescan all lengths before copying. Spans are expressed in output coordinates and + // preserve adjacent surviving runs for consumers that can amortize range setup. + virtual Status consume_plain_byte_array(const char* encoded_data, + const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) { + std::vector values; + values.reserve(num_values); + for (size_t row = 0; row < num_values; ++row) { + values.emplace_back(encoded_data + payload_offsets[row], + value_offsets[row + 1] - value_offsets[row]); + } + return consume(values.data(), values.size()); + } }; // Physical value ranges selected from one page-bounded decode request. Definition-level NULLs are diff --git a/be/src/format/column_type_convert.cpp b/be/src/format/column_type_convert.cpp index 414691d386d201..08fe5c8a4ce794 100644 --- a/be/src/format/column_type_convert.cpp +++ b/be/src/format/column_type_convert.cpp @@ -109,12 +109,7 @@ ColumnPtr ColumnTypeConverter::get_column(const DataTypePtr& src_type, ColumnPtr } if (!_cached_src_column) { - // Projection metadata can be non-nullable while the actual output block keeps a nullable - // wrapper (for example an Iceberg equality-delete key). Mirror the physical destination - // column so decoded null levels always have a temporary null map to propagate. - const bool destination_is_nullable = - dst_type->is_nullable() || is_column_nullable(*dst_column); - _cached_src_type = destination_is_nullable + _cached_src_type = dst_type->is_nullable() ? get_data_type_with_default_argument(make_nullable(src_type)) : get_data_type_with_default_argument(remove_nullable(src_type)); _cached_src_column = _cached_src_type->create_column(); diff --git a/be/src/format/parquet/parquet_thrift_util.h b/be/src/format/parquet/parquet_thrift_util.h index d28b982fea5911..1dbbf8eff20ffd 100644 --- a/be/src/format/parquet/parquet_thrift_util.h +++ b/be/src/format/parquet/parquet_thrift_util.h @@ -39,8 +39,7 @@ constexpr size_t INIT_META_SIZE = 48 * 1024; // 48k static Status parse_thrift_footer(io::FileReaderSPtr file, std::unique_ptr* file_metadata, size_t* meta_size, io::IOContext* io_ctx, const bool enable_mapping_varbinary, - const bool enable_mapping_timestamp_tz, - bool retain_serialized_metadata = false) { + const bool enable_mapping_timestamp_tz) { size_t file_size = file->size(); size_t bytes_read = std::min(file_size, INIT_META_SIZE); std::vector footer(bytes_read); @@ -78,17 +77,10 @@ static Status parse_thrift_footer(io::FileReaderSPtr file, meta_ptr = footer.data() + bytes_read - PARQUET_FOOTER_SIZE - metadata_size; } - std::vector serialized_metadata; - if (retain_serialized_metadata) { - // Small-file v2 opens still need Arrow's page-index adapter. Retaining the bytes already - // fetched from storage avoids serializing the just-decoded Thrift footer a second time. - serialized_metadata.assign(meta_ptr, meta_ptr + metadata_size); - } tparquet::FileMetaData t_metadata; // deserialize footer RETURN_IF_ERROR(deserialize_thrift_msg(meta_ptr, &metadata_size, true, &t_metadata)); - *file_metadata = std::make_unique(t_metadata, metadata_size, - std::move(serialized_metadata)); + *file_metadata = std::make_unique(t_metadata, metadata_size); RETURN_IF_ERROR( (*file_metadata)->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); *meta_size = PARQUET_FOOTER_SIZE + metadata_size; diff --git a/be/src/format/parquet/vparquet_file_metadata.cpp b/be/src/format/parquet/vparquet_file_metadata.cpp index c39a84e1715ca9..e2886ef624ec9e 100644 --- a/be/src/format/parquet/vparquet_file_metadata.cpp +++ b/be/src/format/parquet/vparquet_file_metadata.cpp @@ -18,31 +18,23 @@ #include "format/parquet/vparquet_file_metadata.h" #include -#include -#include #include #include -#include "common/cast_set.h" #include "format/parquet/schema_desc.h" #include "runtime/exec_env.h" #include "runtime/memory/mem_tracker_limiter.h" -#include "util/thrift_util.h" namespace doris { -FileMetaData::FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size, - std::vector serialized_metadata) - : _metadata(metadata), - _mem_size(mem_size), - _serialized_metadata(std::move(serialized_metadata)) { - ExecEnv::GetInstance()->parquet_meta_tracker()->consume(mem_size + _serialized_metadata.size()); +FileMetaData::FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size) + : _metadata(metadata), _mem_size(mem_size) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(mem_size); } FileMetaData::~FileMetaData() { - ExecEnv::GetInstance()->parquet_meta_tracker()->release( - _mem_size + _serialized_metadata.size() + _arrow_metadata_mem_size); + ExecEnv::GetInstance()->parquet_meta_tracker()->release(_mem_size); } Status FileMetaData::init_schema(const bool enable_mapping_varbinary, @@ -52,51 +44,13 @@ Status FileMetaData::init_schema(const bool enable_mapping_varbinary, } _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); - RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); - // Cached metadata is immutable after publication; assign native field IDs before insertion. - _schema.assign_ids(); - return Status::OK(); + return _schema.parse_from_thrift(_metadata.schema); } const tparquet::FileMetaData& FileMetaData::to_thrift() const { return _metadata; } -Status FileMetaData::get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const { - DORIS_CHECK(metadata != nullptr); - std::lock_guard lock(_arrow_metadata_mutex); - if (_arrow_metadata == nullptr) { - try { - std::vector fallback_serialized_metadata; - const std::vector* serialized_metadata = &_serialized_metadata; - if (serialized_metadata->empty()) { - ThriftSerializer serializer(/*compact=*/true, - static_cast(std::max(_mem_size, 4096))); - RETURN_IF_ERROR( - serializer.serialize(const_cast(&_metadata), - &fallback_serialized_metadata)); - serialized_metadata = &fallback_serialized_metadata; - } - uint32_t serialized_size = cast_set(serialized_metadata->size()); - auto parsed = - ::parquet::FileMetaData::Make(serialized_metadata->data(), &serialized_size, - ::parquet::default_reader_properties()); - DORIS_CHECK(static_cast(serialized_size) == serialized_metadata->size()); - // Native and Arrow planners describe the same immutable footer. Cache the adapter at - // the footer-cache lifecycle so repeated v2 opens do not serialize and parse it again. - _arrow_metadata_mem_size = parsed->size(); - ExecEnv::GetInstance()->parquet_meta_tracker()->consume(_arrow_metadata_mem_size); - _arrow_metadata = std::move(parsed); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to adapt cached Parquet metadata: {}", e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to adapt cached Parquet metadata: {}", e.what()); - } - } - *metadata = _arrow_metadata; - return Status::OK(); -} - std::string FileMetaData::debug_string() const { std::stringstream out; out << "Parquet Metadata("; diff --git a/be/src/format/parquet/vparquet_file_metadata.h b/be/src/format/parquet/vparquet_file_metadata.h index 8089e20f1a40be..5d0ba77aced3e9 100644 --- a/be/src/format/parquet/vparquet_file_metadata.h +++ b/be/src/format/parquet/vparquet_file_metadata.h @@ -18,40 +18,27 @@ #pragma once #include -#include -#include #include -#include #include "common/status.h" #include "format/parquet/schema_desc.h" -namespace parquet { -class FileMetaData; -} - namespace doris { class FileMetaData { public: - FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size, - std::vector serialized_metadata = {}); + FileMetaData(tparquet::FileMetaData& metadata, size_t mem_size); ~FileMetaData(); Status init_schema(const bool enable_mapping_varbinary, const bool enable_mapping_timestamp_tz); const FieldDescriptor& schema() const { return _schema; } FieldDescriptor& schema() { return _schema; } const tparquet::FileMetaData& to_thrift() const; - Status get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const; std::string debug_string() const; - size_t get_mem_size() const { return _mem_size + _serialized_metadata.size(); } + size_t get_mem_size() const { return _mem_size; } private: tparquet::FileMetaData _metadata; FieldDescriptor _schema; size_t _mem_size; - mutable std::mutex _arrow_metadata_mutex; - mutable std::shared_ptr<::parquet::FileMetaData> _arrow_metadata; - mutable size_t _arrow_metadata_mem_size = 0; - std::vector _serialized_metadata; }; } // namespace doris diff --git a/be/src/format/parquet/vparquet_reader.cpp b/be/src/format/parquet/vparquet_reader.cpp index 7a8b8f70de072a..26caf7faac66f2 100644 --- a/be/src/format/parquet/vparquet_reader.cpp +++ b/be/src/format/parquet/vparquet_reader.cpp @@ -368,8 +368,7 @@ Status ParquetReader::_open_file() { _reader_statistics.file_footer_read_calls += 1; } else { const auto& file_meta_cache_key = - FileMetaCache::get_key(_tracing_file_reader, _file_description, - enable_mapping_varbinary, enable_mapping_timestamp_tz); + FileMetaCache::get_key(_tracing_file_reader, _file_description); if (!_meta_cache->lookup(file_meta_cache_key, &_meta_cache_handle)) { RETURN_IF_ERROR(parse_thrift_footer(_tracing_file_reader, &_file_metadata_ptr, &meta_size, _io_ctx, enable_mapping_varbinary, diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 69e035b43bc9ff..8c9caddfde12ec 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -33,7 +34,7 @@ #include "common/cast_set.h" #include "common/check.h" #include "common/config.h" -#include "format/parquet/parquet_thrift_util.h" +#include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/cache/cached_remote_file_reader.h" #include "io/file_factory.h" @@ -44,10 +45,69 @@ #include "io/io_common.h" #include "runtime/exec_env.h" #include "storage/cache/page_cache.h" +#include "util/coding.h" #include "util/slice.h" +#include "util/thrift_util.h" +#include "util/time.h" namespace doris::format::parquet { +NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size, + std::vector serialized_metadata) + : _metadata(std::move(metadata)), + _parsed_size(parsed_size), + _serialized_metadata(std::move(serialized_metadata)) { + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); +} + +NativeParquetMetadata::~NativeParquetMetadata() { + ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size() + + _arrow_metadata_mem_size); +} + +Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + _schema.set_enable_mapping_varbinary(enable_mapping_varbinary); + _schema.set_enable_mapping_timestamp_tz(enable_mapping_timestamp_tz); + RETURN_IF_ERROR(_schema.parse_from_thrift(_metadata.schema)); + // Native readers address projected leaves by stable DFS IDs. Assign them only on the private + // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. + _schema.assign_ids(); + return Status::OK(); +} + +Status NativeParquetMetadata::get_arrow_metadata( + std::shared_ptr<::parquet::FileMetaData>* metadata) const { + DORIS_CHECK(metadata != nullptr); + std::lock_guard lock(_arrow_metadata_mutex); + if (_arrow_metadata == nullptr) { + try { + uint32_t serialized_size = cast_set(_serialized_metadata.size()); + auto parsed = + ::parquet::FileMetaData::Make(_serialized_metadata.data(), &serialized_size, + ::parquet::default_reader_properties()); + if (static_cast(serialized_size) != _serialized_metadata.size()) { + return Status::Corruption("Arrow consumed {} of {} Parquet footer bytes", + serialized_size, _serialized_metadata.size()); + } + _arrow_metadata_mem_size = parsed->size(); + ExecEnv::GetInstance()->parquet_meta_tracker()->consume(_arrow_metadata_mem_size); + _arrow_metadata = std::move(parsed); + } catch (const ::parquet::ParquetException& e) { + return Status::Corruption("Failed to adapt v2 Parquet metadata: {}", e.what()); + } catch (const std::exception& e) { + return Status::InternalError("Failed to adapt v2 Parquet metadata: {}", e.what()); + } + } + *metadata = _arrow_metadata; + return Status::OK(); +} + +size_t NativeParquetMetadata::arrow_metadata_mem_size() const { + std::lock_guard lock(_arrow_metadata_mutex); + return _arrow_metadata_mem_size; +} + namespace detail { namespace { @@ -203,6 +263,12 @@ bool should_use_merge_range_reader(const std::vector& ran avg_io_size < io::MergeRangeFileReader::SMALL_IO; } +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size) { + return file_size <= in_memory_file_size && + (path.starts_with("http://") || path.starts_with("https://")); +} + } // namespace detail namespace { @@ -217,6 +283,73 @@ detail::ParquetPageCacheRangeDirectory& cached_page_range_directory() { return directory; } +constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; +constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; +constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; + +Status parse_native_parquet_footer(io::FileReaderSPtr file, + std::unique_ptr* metadata, + size_t* footer_size, io::IOContext* io_ctx, + bool enable_mapping_varbinary, + bool enable_mapping_timestamp_tz) { + DORIS_CHECK(file != nullptr); + DORIS_CHECK(metadata != nullptr); + DORIS_CHECK(footer_size != nullptr); + const size_t file_size = file->size(); + if (file_size < V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 file is too small for a footer: {}", file_size); + } + + const size_t tail_size = std::min(file_size, V2_INITIAL_FOOTER_READ_SIZE); + std::vector tail(tail_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - tail_size, Slice(tail.data(), tail.size()), + &bytes_read, io_ctx)); + if (bytes_read != tail.size()) { + return Status::Corruption("Short Parquet v2 footer read: expected {}, got {}", tail.size(), + bytes_read); + } + const auto* magic = tail.data() + tail.size() - sizeof(V2_PARQUET_MAGIC); + if (memcmp(magic, V2_PARQUET_MAGIC, sizeof(V2_PARQUET_MAGIC)) != 0) { + return Status::Corruption("Invalid Parquet v2 footer magic in {}", file->path().native()); + } + + const uint32_t serialized_size = + decode_fixed32_le(tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE); + if (serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { + // Footer lengths are untrusted. Validate before subtraction/allocation so a malformed + // small file cannot redirect the v2 reader or request an oversized metadata buffer. + return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, + file_size); + } + std::vector serialized_metadata(serialized_size); + if (serialized_size <= tail.size() - V2_PARQUET_FOOTER_SIZE) { + const auto* metadata_start = + tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE - serialized_size; + memcpy(serialized_metadata.data(), metadata_start, serialized_size); + } else { + bytes_read = 0; + RETURN_IF_ERROR(file->read_at(file_size - V2_PARQUET_FOOTER_SIZE - serialized_size, + Slice(serialized_metadata.data(), serialized_metadata.size()), + &bytes_read, io_ctx)); + if (bytes_read != serialized_metadata.size()) { + return Status::Corruption("Short Parquet v2 metadata read: expected {}, got {}", + serialized_metadata.size(), bytes_read); + } + } + + uint32_t thrift_size = serialized_size; + tparquet::FileMetaData thrift_metadata; + RETURN_IF_ERROR(deserialize_thrift_msg(serialized_metadata.data(), &thrift_size, true, + &thrift_metadata)); + auto parsed = std::make_unique( + std::move(thrift_metadata), serialized_size, std::move(serialized_metadata)); + RETURN_IF_ERROR(parsed->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); + *footer_size = V2_PARQUET_FOOTER_SIZE + serialized_size; + *metadata = std::move(parsed); + return Status::OK(); +} + std::string build_page_cache_file_key(const io::FileReader& file_reader, const io::FileDescription& file_description) { const int64_t mtime = @@ -578,50 +711,62 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont bool enable_page_cache, const io::FileDescription& file_description, bool enable_mapping_timestamp_tz) { DORIS_CHECK(input_file_reader != nullptr); - native_file = input_file_reader; + if (detail::should_stage_small_http_file(input_file_reader->path().native(), + input_file_reader->size(), + config::in_memory_file_size)) { + // A metadata-cache hit can make the first physical read start inside a tiny HTTP file. + // Read it from byte zero once so EOF-range quirks cannot make warm scans less reliable + // than cold scans, while keeping this compatibility policy entirely inside v2. + native_file = std::make_shared(std::move(input_file_reader)); + } else { + native_file = std::move(input_file_reader); + } native_io_ctx = io_ctx; - // Use the exact footer cache key and payload type used by v1. This deliberately happens before - // Arrow metadata is opened: native readers can reuse a footer produced by a v1 scan (and vice - // versa), and a cache miss performs one bounded tail read through the same Doris FileReader. + // V2 owns its footer payload and cache identity. Mapping flags affect the parsed schema, and a + // distinct suffix prevents a v1 FileMetaData value from being cast as the v2-owned type. auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); - const auto meta_cache_key = - FileMetaCache::get_key(native_file, file_description, /*enable_mapping_varbinary=*/true, - enable_mapping_timestamp_tz); + auto meta_cache_key = FileMetaCache::get_key(native_file, file_description); + meta_cache_key.append("\0v2", 3); + meta_cache_key.push_back(static_cast(true)); + meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); size_t native_footer_size = 0; if (meta_cache != nullptr && meta_cache->enabled() && meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { - native_metadata = native_meta_cache_handle.data(); + native_metadata = native_meta_cache_handle.data(); ++native_footer_cache_hits; } else { - RETURN_IF_ERROR(parse_thrift_footer( + RETURN_IF_ERROR(parse_native_parquet_footer( native_file, &native_metadata_owner, &native_footer_size, io_ctx, - /*enable_mapping_varbinary=*/true, enable_mapping_timestamp_tz, - /*retain_serialized_metadata=*/true)); + /*enable_mapping_varbinary=*/true, enable_mapping_timestamp_tz)); ++native_footer_read_calls; if (meta_cache != nullptr && meta_cache->enabled()) { meta_cache->insert(meta_cache_key, native_metadata_owner.release(), &native_meta_cache_handle); - native_metadata = native_meta_cache_handle.data(); + native_metadata = native_meta_cache_handle.data(); } else { native_metadata = native_metadata_owner.get(); } } DORIS_CHECK(native_metadata != nullptr); - auto page_cache_file_key = build_page_cache_file_key(*input_file_reader, file_description); + auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); // Native and Arrow readers must use the same FileDescription-derived immutable identity. native_page_cache_file_key = page_cache_file_key; - arrow_file = std::make_shared( - input_file_reader, io_ctx, enable_page_cache, std::move(page_cache_file_key)); + arrow_file = std::make_shared(native_file, io_ctx, enable_page_cache, + std::move(page_cache_file_key)); try { + const int64_t adapter_start_ns = MonotonicNanos(); std::shared_ptr<::parquet::FileMetaData> arrow_metadata; RETURN_IF_ERROR(native_metadata->get_arrow_metadata(&arrow_metadata)); this->file_reader = ::parquet::ParquetFileReader::Open( arrow_file, ::parquet::default_reader_properties(), std::move(arrow_metadata)); metadata = this->file_reader->metadata(); schema = metadata != nullptr ? metadata->schema() : nullptr; + arrow_metadata_adapter_time += MonotonicNanos() - adapter_start_ns; + arrow_metadata_adapter_bytes = + static_cast(native_metadata->arrow_metadata_mem_size()); } catch (const ::parquet::ParquetException& e) { if (io_ctx != nullptr && io_ctx->should_stop && std::string_view(e.what()).find("stop") != std::string_view::npos) { @@ -647,7 +792,7 @@ Status ParquetFileContext::load_native_offset_indexes( std::unordered_map* offset_indexes) const { DORIS_CHECK(offset_indexes != nullptr); offset_indexes->clear(); - if (leaf_column_ids.empty() || file_reader == nullptr) { + if (leaf_column_ids.empty()) { return Status::OK(); } const auto& thrift_metadata = native_metadata->to_thrift(); @@ -658,34 +803,41 @@ Status ParquetFileContext::load_native_offset_indexes( const auto compat = native::parquet_reader_compat( thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); try { - auto page_index_reader = file_reader->GetPageIndexReader(); - if (page_index_reader == nullptr) { - return Status::OK(); - } - auto row_group_reader = page_index_reader->RowGroup(row_group_id); - if (row_group_reader == nullptr) { - return Status::OK(); - } for (const int leaf_column_id : leaf_column_ids) { if (leaf_column_id < 0 || leaf_column_id >= static_cast(native_row_group.columns.size())) { return Status::Corruption("Invalid Parquet leaf {} for OffsetIndex", leaf_column_id); } - auto arrow_index = row_group_reader->GetOffsetIndex(leaf_column_id); - if (arrow_index == nullptr || arrow_index->page_locations().empty()) { - // An empty optional index is equivalent to no index. Publishing it would select - // the indexed PageReader even though there is no first page to dereference. + const auto& column_chunk = native_row_group.columns[leaf_column_id]; + if (!column_chunk.__isset.offset_index_offset || + !column_chunk.__isset.offset_index_length || + column_chunk.offset_index_length <= 0) { + continue; + } + const int64_t index_offset = column_chunk.offset_index_offset; + const int64_t index_length = column_chunk.offset_index_length; + if (index_offset < 0 || index_length <= 0 || index_offset > native_file->size() || + index_length > native_file->size() - index_offset || + index_length > std::numeric_limits::max()) { + // OffsetIndex is optional. A malformed range must not allocate from untrusted + // footer values or redirect the native reader outside the file. + continue; + } + std::vector serialized_index(static_cast(index_length)); + Slice index_slice(serialized_index.data(), serialized_index.size()); + size_t bytes_read = 0; + if (!native_file->read_at(index_offset, index_slice, &bytes_read, native_io_ctx).ok() || + bytes_read != serialized_index.size()) { continue; } + uint32_t thrift_length = static_cast(serialized_index.size()); tparquet::OffsetIndex native_index; - native_index.page_locations.reserve(arrow_index->page_locations().size()); - for (const auto& arrow_location : arrow_index->page_locations()) { - tparquet::PageLocation native_location; - native_location.__set_offset(arrow_location.offset); - native_location.__set_compressed_page_size(arrow_location.compressed_page_size); - native_location.__set_first_row_index(arrow_location.first_row_index); - native_index.page_locations.push_back(std::move(native_location)); + if (!deserialize_thrift_msg(serialized_index.data(), &thrift_length, true, + &native_index) + .ok() || + native_index.page_locations.empty()) { + continue; } native::ColumnChunkRange chunk_range; RETURN_IF_ERROR(native::compute_column_chunk_range( @@ -701,12 +853,149 @@ Status ParquetFileContext::load_native_offset_indexes( } offset_indexes->emplace(leaf_column_id, std::move(native_index)); } - } catch (const ::parquet::ParquetException&) { + } catch (const std::exception&) { // OffsetIndex is optional. Selected logical ranges still enforce correctness, while the // native reader conservatively falls back to sequential page traversal. offset_indexes->clear(); - } catch (const std::exception&) { - offset_indexes->clear(); + } + return Status::OK(); +} + +Status ParquetFileContext::load_native_page_indexes( + int row_group_id, const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, int64_t* read_time, + int64_t* parse_time) const { + DORIS_CHECK(page_indexes != nullptr); + page_indexes->clear(); + if (leaf_column_ids.empty()) { + return Status::OK(); + } + const auto& thrift_metadata = native_metadata->to_thrift(); + if (row_group_id < 0 || row_group_id >= static_cast(thrift_metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group {} for PageIndex", row_group_id); + } + const auto& row_group = thrift_metadata.row_groups[row_group_id]; + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : ""); + + struct SerializedIndexRange { + int leaf_column_id; + int64_t offset; + int64_t length; + }; + struct PendingPageIndex { + NativeParquetPageIndex indexes; + bool has_column_index = false; + bool has_offset_index = false; + }; + std::vector column_index_ranges; + std::vector offset_index_ranges; + std::unordered_map pending_indexes; + + auto valid_index_range = [&](int64_t offset, int64_t length) { + if (offset < 0 || length <= 0 || offset > native_file->size() || + length > native_file->size() - offset || + length > std::numeric_limits::max()) { + return false; + } + return true; + }; + + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + return Status::Corruption("Invalid Parquet leaf {} for PageIndex", leaf_column_id); + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (!chunk.__isset.column_index_offset || !chunk.__isset.column_index_length || + !chunk.__isset.offset_index_offset || !chunk.__isset.offset_index_length) { + continue; + } + if (!valid_index_range(chunk.column_index_offset, chunk.column_index_length) || + !valid_index_range(chunk.offset_index_offset, chunk.offset_index_length)) { + continue; + } + column_index_ranges.push_back( + {leaf_column_id, chunk.column_index_offset, chunk.column_index_length}); + offset_index_ranges.push_back( + {leaf_column_id, chunk.offset_index_offset, chunk.offset_index_length}); + pending_indexes.try_emplace(leaf_column_id); + } + + auto read_coalesced_indexes = [&](std::vector* ranges, + bool column_index) { + std::sort(ranges->begin(), ranges->end(), + [](const auto& lhs, const auto& rhs) { return lhs.offset < rhs.offset; }); + size_t range_begin = 0; + while (range_begin < ranges->size()) { + size_t range_end = range_begin + 1; + int64_t span_end = (*ranges)[range_begin].offset + (*ranges)[range_begin].length; + while (range_end < ranges->size() && (*ranges)[range_end].offset <= span_end) { + span_end = std::max(span_end, + (*ranges)[range_end].offset + (*ranges)[range_end].length); + ++range_end; + } + + const int64_t span_offset = (*ranges)[range_begin].offset; + const int64_t span_length = span_end - span_offset; + std::vector serialized(static_cast(span_length)); + Slice slice(serialized.data(), serialized.size()); + size_t bytes_read = 0; + Status read_status; + int64_t read_time_sink = 0; + { + SCOPED_RAW_TIMER(read_time == nullptr ? &read_time_sink : read_time); + read_status = native_file->read_at(span_offset, slice, &bytes_read, native_io_ctx); + } + if (read_status.ok() && bytes_read == serialized.size()) { + for (size_t i = range_begin; i < range_end; ++i) { + const auto& range = (*ranges)[i]; + auto pending = pending_indexes.find(range.leaf_column_id); + if (pending == pending_indexes.end()) { + continue; + } + uint32_t thrift_length = static_cast(range.length); + const auto* thrift_data = + serialized.data() + static_cast(range.offset - span_offset); + int64_t parse_time_sink = 0; + SCOPED_RAW_TIMER(parse_time == nullptr ? &parse_time_sink : parse_time); + if (column_index) { + pending->second.has_column_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.column_index) + .ok(); + } else { + pending->second.has_offset_index = + deserialize_thrift_msg(thrift_data, &thrift_length, true, + &pending->second.indexes.offset_index) + .ok(); + } + } + } + range_begin = range_end; + } + }; + + // Parquet writers place each index kind in a contiguous block. Reading overlapping/adjacent + // ranges as one span keeps cold small-file planning from paying two remote round trips per + // projected leaf, while refusing gaps avoids amplifying reads from untrusted footer offsets. + read_coalesced_indexes(&column_index_ranges, true); + read_coalesced_indexes(&offset_index_ranges, false); + + for (auto& [leaf_column_id, pending] : pending_indexes) { + const auto& chunk = row_group.columns[leaf_column_id]; + auto& indexes = pending.indexes; + if (!pending.has_column_index || !pending.has_offset_index || + indexes.column_index.null_pages.size() != indexes.offset_index.page_locations.size()) { + continue; + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, native_file->size(), compat.parquet_816_padding, &chunk_range)); + if (!native::validate_offset_index(indexes.offset_index, chunk_range, + chunk.meta_data.data_page_offset, row_group.num_rows)) { + continue; + } + page_indexes->emplace(leaf_column_id, std::move(indexes)); } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index c1457547003a1c..5e62becfb81e29 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -23,12 +24,13 @@ #include #include #include +#include #include #include #include #include "common/status.h" -#include "format/parquet/vparquet_file_metadata.h" +#include "format/parquet/schema_desc.h" #include "io/fs/file_reader.h" #include "util/obj_lru_cache.h" @@ -43,6 +45,33 @@ class RuntimeProfile; namespace doris::format::parquet { +struct NativeParquetPageIndex; + +// V2-owned footer/schema object. Keeping this adapter here prevents native scanning requirements +// from changing the established v1 FileMetaData cache value or parser behavior. +class NativeParquetMetadata { +public: + NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size, + std::vector serialized_metadata); + ~NativeParquetMetadata(); + + Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); + const tparquet::FileMetaData& to_thrift() const { return _metadata; } + const FieldDescriptor& schema() const { return _schema; } + Status get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const; + size_t arrow_metadata_mem_size() const; + size_t get_mem_size() const { return _parsed_size + _serialized_metadata.size(); } + +private: + tparquet::FileMetaData _metadata; + FieldDescriptor _schema; + size_t _parsed_size = 0; + std::vector _serialized_metadata; + mutable std::mutex _arrow_metadata_mutex; + mutable std::shared_ptr<::parquet::FileMetaData> _arrow_metadata; + mutable size_t _arrow_metadata_mem_size = 0; +}; + struct ParquetPageCacheRange { int64_t offset = 0; int64_t size = 0; @@ -142,6 +171,11 @@ size_t average_prefetch_range_size(const std::vector& ran bool should_use_merge_range_reader(const std::vector& ranges, size_t avg_io_size, bool is_in_memory_reader); +// HTTP range servers may reject an overlong range near EOF even after accepting the capability +// probe. Staging a bounded small HTTP object also turns all native page reads into memory copies. +bool should_stage_small_http_file(std::string_view path, size_t file_size, + size_t in_memory_file_size); + } // namespace detail struct ParquetFileContext { @@ -153,14 +187,15 @@ struct ParquetFileContext { // MergeRangeFileReader policy as v1; large chunks and in-memory files keep native_file. io::FileReaderSPtr native_row_group_file; io::IOContext* native_io_ctx = nullptr; - // V1-compatible Thrift footer/schema used to construct Doris' native page/encoding readers. - // A cache hit is owned by native_meta_cache_handle; a miss without cache is owned by - // native_metadata_owner. - const FileMetaData* native_metadata = nullptr; - std::unique_ptr native_metadata_owner; + // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is + // owned by native_meta_cache_handle; a miss without cache is owned by native_metadata_owner. + const NativeParquetMetadata* native_metadata = nullptr; + std::unique_ptr native_metadata_owner; ObjLRUCache::CacheHandle native_meta_cache_handle; int64_t native_footer_read_calls = 0; int64_t native_footer_cache_hits = 0; + int64_t arrow_metadata_adapter_time = 0; + int64_t arrow_metadata_adapter_bytes = 0; bool native_page_cache_enabled = false; std::string native_page_cache_file_key; @@ -175,6 +210,11 @@ struct ParquetFileContext { Status load_native_offset_indexes( int row_group_id, const std::unordered_set& leaf_column_ids, std::unordered_map* offset_indexes) const; + Status load_native_page_indexes(int row_group_id, + const std::unordered_set& leaf_column_ids, + std::unordered_map* page_indexes, + int64_t* read_time = nullptr, + int64_t* parse_time = nullptr) const; // Register ranges for the remaining Arrow metadata/index adapter. Native data pages use the // v1-compatible page cache owned by BufferedFileStreamReader instead. void register_page_cache_ranges(std::vector ranges); diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 8994c32d3d4036..a5f0cd51050f7c 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -106,6 +106,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); parse_meta_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseMetaTime", parquet_profile, 1); + arrow_metadata_adapter_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowMetadataAdapterTime", parquet_profile, 1); + arrow_metadata_adapter_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "ArrowMetadataAdapterBytes", TUnit::BYTES, parquet_profile, 1); parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); file_reader_create_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); @@ -165,6 +169,12 @@ void ParquetProfile::init(RuntimeProfile* profile) { parquet_profile, 1); predicate_filter_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateFilterTime", parquet_profile, 1); + predicate_compaction_time = + ADD_CHILD_TIMER_WITH_LEVEL(profile, "PredicateCompactionTime", parquet_profile, 1); + predicate_compaction_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionBytes", + TUnit::BYTES, parquet_profile, 1); + predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", + TUnit::UNIT, parquet_profile, 1); dict_filter_rewrite_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); dict_filter_expr_rewrite_time = @@ -273,6 +283,9 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .range_gap_skipped_rows = range_gap_skipped_rows, .column_read_time = column_read_time, .predicate_filter_time = predicate_filter_time, + .predicate_compaction_time = predicate_compaction_time, + .predicate_compaction_bytes = predicate_compaction_bytes, + .predicate_compaction_count = predicate_compaction_count, .dict_filter_rewrite_time = dict_filter_rewrite_time, .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, .dict_filter_read_dict_time = dict_filter_read_dict_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 1b11001426b560..28b63eacd24ab5 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -87,6 +87,9 @@ struct ParquetScanProfile { RuntimeProfile::Counter* range_gap_skipped_rows = nullptr; // rows skipped by range gaps RuntimeProfile::Counter* column_read_time = nullptr; // column read time (ns) RuntimeProfile::Counter* predicate_filter_time = nullptr; // predicate filter time (ns) + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; // expression/residual rewrite time (ns) @@ -159,6 +162,8 @@ struct ParquetProfile { RuntimeProfile::Counter* column_read_time = nullptr; RuntimeProfile::Counter* parse_meta_time = nullptr; + RuntimeProfile::Counter* arrow_metadata_adapter_time = nullptr; + RuntimeProfile::Counter* arrow_metadata_adapter_bytes = nullptr; RuntimeProfile::Counter* parse_footer_time = nullptr; RuntimeProfile::Counter* file_reader_create_time = nullptr; RuntimeProfile::Counter* open_file_num = nullptr; @@ -195,6 +200,9 @@ struct ParquetProfile { RuntimeProfile::Counter* parse_page_header_num = nullptr; RuntimeProfile::Counter* predicate_filter_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_time = nullptr; + RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; + RuntimeProfile::Counter* predicate_compaction_count = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 649d3e2b54989d..21828e2a601891 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -436,6 +436,10 @@ Status ParquetReader::init(RuntimeState* state) { _state->file_context.native_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _state->file_context.native_footer_cache_hits); + COUNTER_UPDATE(_parquet_profile.arrow_metadata_adapter_time, + _state->file_context.arrow_metadata_adapter_time); + COUNTER_UPDATE(_parquet_profile.arrow_metadata_adapter_bytes, + _state->file_context.arrow_metadata_adapter_bytes); } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier @@ -545,7 +549,7 @@ Status ParquetReader::open(std::shared_ptr request) { RETURN_IF_ERROR(plan_parquet_row_groups( *_state->file_context.metadata, _state->file_context.file_reader.get(), _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, - &row_group_plan, _state->timezone, _state->runtime_state)); + &row_group_plan, _state->timezone, _state->runtime_state, &_state->file_context)); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 199a6aab7fb076..8a38cc4a1c457c 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -87,10 +87,19 @@ std::vector adaptive_prefetch_prefix( return result; } +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { + constexpr size_t WARMUP_SAMPLES = 8; + constexpr size_t STEADY_STATE_INTERVAL = 16; + return samples < WARMUP_SAMPLES || batch_sequence % STEADY_STATE_INTERVAL == 0; +} + } // namespace detail namespace { +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( + const format::FileScanRequest& request); + int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { return column_metadata.has_dictionary_page() ? cast_set(column_metadata.dictionary_page_offset()) @@ -335,9 +344,21 @@ Status build_row_group_read_plans( const std::vector>& file_schema, const format::FileScanRequest& request, const std::vector& selected_row_groups, const std::vector& row_group_first_rows, RowGroupScanPlan* plan, - const cctz::time_zone* timezone, const RuntimeState* runtime_state) { + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { DORIS_CHECK(plan != nullptr); plan->row_groups.reserve(selected_row_groups.size()); + std::unordered_set requested_leaf_ids; + if (file_context != nullptr) { + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; + } + collect_projected_leaf_column_ids(*file_schema[local_id], projection, + &requested_leaf_ids); + } + } for (const auto row_group_idx : selected_row_groups) { DORIS_CHECK(row_group_idx >= 0); DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); @@ -352,10 +373,28 @@ Status build_row_group_read_plans( row_group_plan.row_group_id = row_group_idx; row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; row_group_plan.row_group_rows = row_group_rows; - RETURN_IF_ERROR(select_row_group_ranges_by_page_index( - file_reader, file_schema, request, row_group_idx, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); + if (file_context != nullptr) { + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_idx, requested_leaf_ids, &page_indexes, + &plan->pruning_stats.read_page_index_time, + &plan->pruning_stats.parse_page_index_time)); + } + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + page_indexes, file_schema, request, row_group_rows, + &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, + &plan->pruning_stats, timezone, runtime_state)); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan.offset_indexes.emplace(leaf_column_id, + std::move(indexes.offset_index)); + } + } else { + RETURN_IF_ERROR(select_row_group_ranges_by_page_index( + file_reader, file_schema, request, row_group_idx, row_group_rows, + &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, + &plan->pruning_stats, timezone, runtime_state)); + } if (row_group_plan.selected_ranges.empty()) { continue; } @@ -373,7 +412,8 @@ Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { + const RuntimeState* runtime_state, + ParquetFileContext* file_context) { DORIS_CHECK(plan != nullptr); plan->row_groups.clear(); plan->pruning_stats = ParquetPruningStats {}; @@ -412,7 +452,7 @@ Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, metadata_selected_row_groups, row_group_first_rows, - plan, timezone, runtime_state)); + plan, timezone, runtime_state, file_context)); plan->pruning_stats.selected_row_groups = plan->row_groups.size(); return Status::OK(); } @@ -720,10 +760,21 @@ void ParquetScanScheduler::set_condition_cache_context(std::shared_ptrsecond.value(); + _predicate_positions_scratch.push_back(position); + _predicate_indices_by_position_scratch.emplace(position, idx); + } + return _predicate_schedule; +} + std::vector ParquetScanScheduler::adaptive_predicate_prefetch_columns( const format::FileScanRequest& request) const { std::vector positions; @@ -793,7 +869,7 @@ Status ParquetScanScheduler::open_next_row_group( _current_merge_range_active = false; return Status::OK(); } - const RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; const int row_group_idx = row_group_plan.row_group_id; // Row-level dictionary filters still use the migration metadata/page probe. Native data-page // readers below do not construct an Arrow RowGroupReader or RecordReader. @@ -811,24 +887,10 @@ Status ParquetScanScheduler::open_next_row_group( _current_row_group_first_row = row_group_plan.first_file_row; _current_row_group_rows_read = 0; _current_selected_ranges = row_group_plan.selected_ranges; - const bool has_filtered_ranges = _current_selected_ranges.size() != 1 || - _current_selected_ranges[0].start != 0 || - _current_selected_ranges[0].length != _current_row_group_rows; - if (has_filtered_ranges) { - std::unordered_set leaf_column_ids; - for (const auto& projection : request_scan_columns(request)) { - const auto local_id = projection.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID || - local_id == format::GLOBAL_ROWID_COLUMN_ID) { - continue; - } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); - collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); - } - RETURN_IF_ERROR(file_context.load_native_offset_indexes(row_group_idx, leaf_column_ids, - &_current_offset_indexes)); - } + _current_offset_indexes = std::move(row_group_plan.offset_indexes); + // Condition Cache and split planning can narrow logical ranges without a physical OffsetIndex. + // Native readers must keep the sequential level/value cursor path valid in that case; only a + // PageIndex-derived skip plan requires the transferred indexes below. for (const auto& [leaf_column_id, skip_plan] : row_group_plan.page_skip_plans) { if (!_current_offset_indexes.contains(leaf_column_id)) { continue; @@ -978,12 +1040,7 @@ Status ParquetScanScheduler::flush_pending_non_predicate_skip_rows() { namespace { -struct PredicateConjunctSchedule { - std::map single_column_conjuncts; - VExprContextSPtrs remaining_conjuncts; -}; - -PredicateConjunctSchedule build_predicate_conjunct_schedule( +detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request) { std::unordered_set predicate_block_positions; predicate_block_positions.reserve(request.predicate_columns.size()); @@ -993,7 +1050,7 @@ PredicateConjunctSchedule build_predicate_conjunct_schedule( predicate_block_positions.insert(position_it->second.value()); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; for (const auto& conjunct : request.conjuncts) { DORIS_CHECK(conjunct != nullptr); DORIS_CHECK(conjunct->root() != nullptr); @@ -1091,15 +1148,6 @@ uint16_t count_selected_rows(const IColumn::Filter& filter) { return selected_rows; } -Status filter_read_predicate_columns(Block* file_block, const std::vector& positions, - const IColumn::Filter& compact_filter) { - if (positions.empty()) { - return Status::OK(); - } - RETURN_IF_CATCH_EXCEPTION(Block::filter_block_internal(file_block, positions, compact_filter)); - return Status::OK(); -} - IColumn::Filter build_dictionary_entry_filter(size_t block_position, const ParquetColumnSchema& column_schema, const VExprContextSPtrs& conjuncts, @@ -1137,10 +1185,10 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( if (request.conjuncts.empty()) { return Status::OK(); } - PredicateConjunctSchedule schedule; + detail::PredicateConjunctSchedule schedule; { SCOPED_TIMER(_scan_profile.dict_filter_expr_rewrite_time); - schedule = build_predicate_conjunct_schedule(request); + schedule = predicate_conjunct_schedule(request); } if (schedule.single_column_conjuncts.empty()) { return Status::OK(); @@ -1233,12 +1281,81 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, if (!request.conjuncts.empty() || !request.delete_conjuncts.empty()) { selection->resize(static_cast(batch_rows)); } - const auto schedule = build_predicate_conjunct_schedule(request); + const auto& schedule = predicate_conjunct_schedule(request); + const size_t predicate_batch_sequence = _predicate_batch_sequence++; const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); auto& read_column_positions = _read_column_positions_scratch; read_column_positions.clear(); read_column_positions.reserve(request.predicate_columns.size()); + for (auto& rows : _predicate_column_selection_scratch | std::views::values) { + rows.clear(); + } + + auto remember_column_selection = [&](uint32_t position) { + auto& rows = _predicate_column_selection_scratch[position]; + rows.resize(*selected_rows); + for (uint16_t row = 0; row < *selected_rows; ++row) { + // SelectionVector and the scanner batch contract both bound row ordinals to uint16_t; + // keep the checked conversion explicit when persisting the coordinate mapping. + rows[row] = cast_set(selection->get_index(row)); + } + }; + + auto compact_predicate_columns = [&]() -> Status { + bool compacted = false; + int64_t compacted_bytes = 0; + for (const uint32_t position : read_column_positions) { + auto& source_rows = _predicate_column_selection_scratch[position]; + const auto& old_column = file_block->get_by_position(position).column; + if (old_column->size() != source_rows.size()) { + return Status::Corruption( + "Predicate column {} has {} values but {} remembered source rows", position, + old_column->size(), source_rows.size()); + } + bool already_compact = source_rows.size() == *selected_rows && + old_column->size() == static_cast(*selected_rows); + for (uint16_t row = 0; already_compact && row < *selected_rows; ++row) { + already_compact = source_rows[row] == selection->get_index(row); + } + if (already_compact) { + continue; + } + auto& filter = _predicate_compaction_filter_scratch; + // resize_fill() preserves bytes when the next predicate column is smaller. Clear the + // whole reusable mask so survivors from an earlier coordinate space cannot reappear. + filter.resize(source_rows.size()); + std::ranges::fill(filter, 0); + size_t source_idx = 0; + uint16_t selected_idx = 0; + while (source_idx < source_rows.size() && selected_idx < *selected_rows) { + const auto source_row = source_rows[source_idx]; + const auto selected_row = selection->get_index(selected_idx); + if (source_row < selected_row) { + ++source_idx; + continue; + } + DORIS_CHECK_EQ(source_row, selected_row); + filter[source_idx++] = 1; + ++selected_idx; + } + DORIS_CHECK_EQ(selected_idx, *selected_rows); + compacted_bytes += static_cast(old_column->byte_size()); + RETURN_IF_CATCH_EXCEPTION(file_block->replace_by_position( + position, old_column->filter(filter, *selected_rows))); + remember_column_selection(position); + compacted = true; + } + if (compacted) { + update_counter_if_not_null(_scan_profile.predicate_compaction_bytes, compacted_bytes); + update_counter_if_not_null(_scan_profile.predicate_compaction_count, 1); + } + // The output path must not apply a batch-coordinate filter to columns that now use compact + // coordinates. The loop above establishes this invariant even when no bytes moved because + // every column was already aligned. + *predicate_columns_filtered = !read_column_positions.empty(); + return Status::OK(); + }; auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, ColumnId local_id, bool* used_dictionary_filter) -> Status { @@ -1270,17 +1387,15 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, update_counter_if_not_null(_scan_profile.rows_filtered_by_dict_filter, filtered_rows); if (new_selected_rows != selected_rows_before) { - // The dictionary reader has already appended only surviving values for the - // current column. Apply the compact row filter only to columns read before this - // one, then update the shared selection for later predicate/lazy columns. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); + // The dictionary reader already appended only survivors for this column. Keep + // older predicate columns in their original coordinate spaces and compact all + // of them once at the expression/output boundary below. *selected_rows = apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); *used_dictionary_filter = true; return Status::OK(); } @@ -1306,6 +1421,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } file_block->replace_by_position(block_position, std::move(column)); read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); return Status::OK(); }; @@ -1327,16 +1443,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // All columns read so far are already compacted to the current selection. Apply the - // compact filter to those columns and the selection vector together, so later predicate - // columns can read only rows that survived previous predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1360,16 +1470,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, static_cast(new_selected_rows); } if (new_selected_rows != selected_rows_before) { - // Dictionary-covered children have already reduced the compact block. Apply only the - // residual child filters here, then keep the same compacted-column invariant as the - // normal conjunct path for later predicate rounds. - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1406,13 +1510,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, compact_filter.resize_fill(selected_rows_before, 0); } if (can_filter_all || count_selected_rows(compact_filter) != selected_rows_before) { - RETURN_IF_ERROR(filter_read_predicate_columns(file_block, read_column_positions, - compact_filter)); *selected_rows = can_filter_all ? 0 : apply_compact_filter_to_selection(compact_filter, selection, selected_rows_before); - *predicate_columns_filtered = true; } return Status::OK(); }; @@ -1443,23 +1544,12 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, // Single-column conjuncts can be evaluated immediately after their column is read. Once // selection shrinks, later predicate columns use ParquetColumnReader::select() so the // reader skips rows already rejected by earlier predicates instead of materializing them. - std::vector positions; - std::unordered_map indices_by_position; - positions.reserve(request.predicate_columns.size()); - indices_by_position.reserve(request.predicate_columns.size()); - for (size_t idx = 0; idx < request.predicate_columns.size(); ++idx) { - const auto position_it = - request.local_positions.find(request.predicate_columns[idx].column_id()); - DORIS_CHECK(position_it != request.local_positions.end()); - const size_t position = position_it->second.value(); - positions.push_back(position); - indices_by_position.emplace(position, idx); - } - const auto ordered_positions = - detail::order_adaptive_predicates(positions, _predicate_runtime_stats); + _ordered_predicate_positions_scratch = detail::order_adaptive_predicates( + _predicate_positions_scratch, _predicate_runtime_stats); + const auto& ordered_positions = _ordered_predicate_positions_scratch; for (size_t order_idx = 0; order_idx < ordered_positions.size(); ++order_idx) { const size_t position = ordered_positions[order_idx]; - const size_t idx = indices_by_position.at(position); + const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; const auto fid = col.local_id(); auto reader_it = _current_predicate_columns.find(fid); @@ -1468,7 +1558,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); const uint16_t rows_before = *selected_rows; - const int64_t start_ns = MonotonicNanos(); + auto& stats = _predicate_runtime_stats[position]; + const bool sample = detail::should_sample_adaptive_predicate(stats.samples, + predicate_batch_sequence); + const int64_t start_ns = sample ? MonotonicNanos() : 0; bool used_dictionary_filter = false; RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, &used_dictionary_filter)); @@ -1483,29 +1576,31 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); } } - auto& stats = _predicate_runtime_stats[position]; - const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / - std::max(rows_before, 1); - const double survival = - static_cast(*selected_rows) / std::max(rows_before, 1); - constexpr double ADAPTIVE_ALPHA = 0.25; - if (stats.samples == 0) { - stats.cost_per_input_row_ns = cost_per_row; - stats.survival_ratio = survival; - } else { - stats.cost_per_input_row_ns = ADAPTIVE_ALPHA * cost_per_row + - (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; - stats.survival_ratio = - ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; + if (sample) { + const double cost_per_row = static_cast(MonotonicNanos() - start_ns) / + std::max(rows_before, 1); + const double survival = + static_cast(*selected_rows) / std::max(rows_before, 1); + constexpr double ADAPTIVE_ALPHA = 0.25; + if (stats.samples == 0) { + stats.cost_per_input_row_ns = cost_per_row; + stats.survival_ratio = survival; + } else { + stats.cost_per_input_row_ns = + ADAPTIVE_ALPHA * cost_per_row + + (1 - ADAPTIVE_ALPHA) * stats.cost_per_input_row_ns; + stats.survival_ratio = + ADAPTIVE_ALPHA * survival + (1 - ADAPTIVE_ALPHA) * stats.survival_ratio; + } + ++stats.samples; } - ++stats.samples; if (*selected_rows != 0) { continue; } for (size_t remaining_order_idx = order_idx + 1; remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { - const size_t remaining_idx = - indices_by_position.at(ordered_positions[remaining_order_idx]); + const size_t remaining_idx = _predicate_indices_by_position_scratch.at( + ordered_positions[remaining_order_idx]); const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); @@ -1516,13 +1611,27 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; + auto compact_predicate_columns_with_profile = [&]() -> Status { + const int64_t start_ns = MonotonicNanos(); + auto status = compact_predicate_columns(); + update_counter_if_not_null(_scan_profile.predicate_compaction_time, + MonotonicNanos() - start_ns); + return status; + }; + RETURN_IF_ERROR(read_round_by_round()); + // Single-column expressions only touch the just-read column, so earlier columns can retain + // their own row mappings. Compact once before a multi-column/output boundary. + RETURN_IF_ERROR(compact_predicate_columns_with_profile()); RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); + RETURN_IF_ERROR(compact_predicate_columns_with_profile()); if (_scan_profile.predicate_filter_time == nullptr) { - return execute_scheduled_delete_conjuncts(); + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); + } else { + SCOPED_TIMER(_scan_profile.predicate_filter_time); + RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } - SCOPED_TIMER(_scan_profile.predicate_filter_time); - return execute_scheduled_delete_conjuncts(); + return compact_predicate_columns_with_profile(); } void ParquetScanScheduler::prefetch_current_row_group_columns( @@ -1550,10 +1659,18 @@ Status ParquetScanScheduler::read_current_row_group_batch( const std::vector>& file_schema, int64_t batch_rows, const format::FileScanRequest& request, int64_t batch_first_file_row, Block* file_block, size_t* rows) { - // All native work performed by this batch, including early empty-selection exits, is published - // together. This preserves slow-path attribution without recursively copying statistics after - // every read/select/skip call. - Defer profile_flush {[this]() { flush_current_reader_profiles(); }}; + // Reader statistics are cumulative plain integers. Publishing their delta recursively for + // every tiny batch is measurable on wide/nested scans, so flush periodically and force the + // tail at row-group reset/close. + Defer profile_flush {[this, batch_rows]() { + const bool finishes_row_group = _current_range_idx + 1 == _current_selected_ranges.size() && + _current_range_rows_read + batch_rows == + _current_selected_ranges[_current_range_idx].length; + if (++_batches_since_profile_flush >= PROFILE_FLUSH_BATCH_INTERVAL || finishes_row_group) { + flush_current_reader_profiles(); + _batches_since_profile_flush = 0; + } + }}; if (_scan_profile.total_batches != nullptr) { COUNTER_UPDATE(_scan_profile.total_batches, 1); } diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 7ca51c656fe9d8..9af54f261c683f 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -62,6 +62,11 @@ struct ParquetFileContext; struct ParquetColumnSchema; namespace detail { +struct PredicateConjunctSchedule { + std::map single_column_conjuncts; + VExprContextSPtrs remaining_conjuncts; +}; + struct AdaptivePredicateStats { double cost_per_input_row_ns = 0; double survival_ratio = 1; @@ -75,6 +80,7 @@ std::vector adaptive_prefetch_prefix( const std::vector& ordered_positions, const std::unordered_map& stats, double minimum_reach_probability); +bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); } // namespace detail // ============================================================================ @@ -93,6 +99,9 @@ struct RowGroupReadPlan { std::vector selected_ranges; // row ranges to read after page-index pruning std::map page_skip_plans; // leaf_column_id -> data pages that can be skipped completely + // Native planning already parsed these indexes. Transfer them to execution so narrowed scans + // do not issue the same remote index reads a second time while opening the row group. + std::unordered_map offset_indexes; }; struct RowGroupScanPlan { @@ -109,7 +118,8 @@ Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); + const RuntimeState* runtime_state = nullptr, + ParquetFileContext* file_context = nullptr); IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -168,8 +178,12 @@ class ParquetScanScheduler { bool* eof); private: + static constexpr size_t PROFILE_FLUSH_BATCH_INTERVAL = 16; + void reset_current_row_group(); void flush_current_reader_profiles(); + const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( + const format::FileScanRequest& request); std::vector adaptive_predicate_prefetch_columns( const format::FileScanRequest& request) const; @@ -248,6 +262,16 @@ class ParquetScanScheduler { // reallocating selection indices, dense filter bytes, or compacted-column positions. SelectionVector _selection; std::vector _read_column_positions_scratch; + const format::FileScanRequest* _predicate_schedule_request = nullptr; + detail::PredicateConjunctSchedule _predicate_schedule; + std::vector _predicate_positions_scratch; + std::unordered_map _predicate_indices_by_position_scratch; + std::vector _ordered_predicate_positions_scratch; + std::unordered_map> + _predicate_column_selection_scratch; + IColumn::Filter _predicate_compaction_filter_scratch; + size_t _predicate_batch_sequence = 0; + size_t _batches_since_profile_flush = 0; std::unordered_map _predicate_runtime_stats; double _predicate_survival_ratio = -1; std::shared_ptr _condition_cache_ctx; diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index ac4bf1d941a126..dd7287300569df 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -52,6 +52,7 @@ #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" +#include "util/unaligned.h" namespace doris::format::parquet { @@ -762,6 +763,11 @@ void accumulate_zonemap_stats(const ZoneMapEvalContext& ctx, ParquetPruningStats } // namespace +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state) { + return config::enable_parquet_page_index && has_expr_zonemap_filter(request, runtime_state); +} + std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( const ParquetColumnStatistics& statistics) { return make_zonemap_from_statistics(statistics); @@ -1626,4 +1632,238 @@ Status select_row_group_ranges_by_page_index( return Status::OK(); } +namespace { + +template +bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + DecodedValueKind kind, ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size() || + column_index.min_values[page_idx].size() != sizeof(ValueType) || + column_index.max_values[page_idx].size() != sizeof(ValueType)) { + return false; + } + const auto min_value = unaligned_load(column_index.min_values[page_idx].data()); + const auto max_value = unaligned_load(column_index.max_values[page_idx].data()); + if constexpr (std::is_same_v) { + if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { + return false; + } + } + if (!valid_min_max(min_value, max_value)) { + return true; + } + if (!set_decoded_field(column_schema, kind, min_value, &page_statistics->min_value, timezone) || + !set_decoded_field(column_schema, kind, max_value, &page_statistics->max_value, timezone)) { + return false; + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; +} + +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone) { + DORIS_CHECK(page_statistics != nullptr); + *page_statistics = {}; + if (!column_index.__isset.null_counts || page_idx >= column_index.null_pages.size() || + page_idx >= column_index.null_counts.size()) { + return false; + } + page_statistics->has_null_count = true; + page_statistics->has_null = column_index.null_counts[page_idx] > 0; + page_statistics->has_not_null = !column_index.null_pages[page_idx]; + if (!page_statistics->has_not_null) { + return true; + } + switch (column_schema.descriptor->physical_type()) { + case ::parquet::Type::BOOLEAN: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::BOOL, page_statistics, + timezone); + case ::parquet::Type::INT32: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case ::parquet::Type::INT64: + return set_native_page_scalar_min_max( + column_index, column_schema, page_idx, + decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); + case ::parquet::Type::FLOAT: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::FLOAT, page_statistics, + timezone); + case ::parquet::Type::DOUBLE: + return set_native_page_scalar_min_max(column_index, column_schema, page_idx, + DecodedValueKind::DOUBLE, page_statistics, + timezone); + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { + if (page_idx >= column_index.min_values.size() || + page_idx >= column_index.max_values.size()) { + return false; + } + const auto& min_value = column_index.min_values[page_idx]; + const auto& max_value = column_index.max_values[page_idx]; + const bool fixed = + column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + if (fixed && + (column_schema.descriptor->type_length() <= 0 || + min_value.size() != static_cast(column_schema.descriptor->type_length()) || + max_value.size() != static_cast(column_schema.descriptor->type_length()))) { + return false; + } + const auto kind = fixed ? DecodedValueKind::FIXED_BINARY : DecodedValueKind::BINARY; + if (!set_decoded_binary_field(column_schema, kind, + StringRef(min_value.data(), min_value.size()), + &page_statistics->min_value, timezone) || + !set_decoded_binary_field(column_schema, kind, + StringRef(max_value.data(), max_value.size()), + &page_statistics->max_value, timezone)) { + return false; + } + if (decoded_min_max_is_ordered(*page_statistics)) { + page_statistics->has_min_max = true; + } + return true; + } + default: + return false; + } +} + +RowRange native_page_row_range(const tparquet::OffsetIndex& offset_index, size_t page_idx, + int64_t row_group_rows) { + const auto& locations = offset_index.page_locations; + const int64_t start = locations[page_idx].first_row_index; + const int64_t end = page_idx + 1 == locations.size() ? row_group_rows + : locations[page_idx + 1].first_row_index; + return {.start = start, .length = end - start}; +} + +} // namespace + +Status select_row_group_ranges_by_native_page_index( + const std::unordered_map& page_indexes, + const std::vector>& file_schema, + const format::FileScanRequest& request, int64_t row_group_rows, + std::vector* selected_ranges, std::map* page_skip_plans, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state) { + int64_t filter_time_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &filter_time_sink + : &pruning_stats->page_index_filter_time); + DORIS_CHECK(selected_ranges != nullptr); + selected_ranges->clear(); + selected_ranges->push_back({.start = 0, .length = row_group_rows}); + if (page_skip_plans != nullptr) { + page_skip_plans->clear(); + } + if (row_group_rows <= 0 || !config::enable_parquet_page_index || + !has_expr_zonemap_filter(request, runtime_state) || page_indexes.empty()) { + return Status::OK(); + } + if (pruning_stats != nullptr) { + ++pruning_stats->page_index_read_calls; + } + + std::map conjuncts_by_slot; + for (const auto& conjunct : request.conjuncts) { + const auto slot_index = expr_zonemap::single_slot_zonemap_index(conjunct); + if (slot_index >= 0) { + conjuncts_by_slot[slot_index].push_back(conjunct); + } + } + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->descriptor == nullptr) { + continue; + } + const auto index_it = page_indexes.find(column_schema->leaf_column_id); + if (index_it == page_indexes.end()) { + continue; + } + const auto& indexes = index_it->second; + std::vector filter_ranges; + bool usable = true; + for (size_t page_idx = 0; page_idx < indexes.offset_index.page_locations.size(); + ++page_idx) { + ParquetColumnStatistics statistics; + if (!build_native_page_statistics(indexes.column_index, *column_schema, page_idx, + &statistics, timezone)) { + usable = false; + break; + } + ZoneMapEvalContext ctx; + add_slot_zonemap(&ctx, slot_index, column_schema->type, + ParquetStatisticsUtils::MakeZoneMap(statistics)); + if (VExprContext::evaluate_zonemap_filter(conjuncts, ctx) != + ZoneMapFilterResult::kNoMatch) { + append_row_range( + native_page_row_range(indexes.offset_index, page_idx, row_group_rows), + &filter_ranges); + } + if (pruning_stats != nullptr) { + pruning_stats->expr_zonemap_unusable_evals += ctx.stats.unusable_zonemap_eval_count; + pruning_stats->in_zonemap_point_check_count += + ctx.stats.in_zonemap_point_check_count; + pruning_stats->in_zonemap_range_only_count += ctx.stats.in_zonemap_range_only_count; + } + } + if (!usable) { + continue; + } + *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); + if (selected_ranges->empty()) { + if (pruning_stats != nullptr) { + pruning_stats->filtered_page_rows += row_group_rows; + ++pruning_stats->filtered_row_groups_by_page_index; + } + return Status::OK(); + } + } + + if (page_skip_plans != nullptr) { + std::vector leaves; + collect_request_leaf_schemas(file_schema, request, &leaves); + for (const auto* leaf : leaves) { + const auto index_it = page_indexes.find(leaf->leaf_column_id); + if (index_it == page_indexes.end() || leaf->descriptor == nullptr || + leaf->descriptor->max_repetition_level() != 0) { + continue; + } + const auto& offset_index = index_it->second.offset_index; + ParquetPageSkipPlan skip_plan; + skip_plan.leaf_column_id = leaf->leaf_column_id; + skip_plan.skipped_pages.resize(offset_index.page_locations.size()); + skip_plan.skipped_page_compressed_sizes.resize(offset_index.page_locations.size()); + for (size_t page_idx = 0; page_idx < offset_index.page_locations.size(); ++page_idx) { + const auto range = native_page_row_range(offset_index, page_idx, row_group_rows); + if (range.length == 0 || ranges_intersect(*selected_ranges, range)) { + continue; + } + skip_plan.skipped_pages[page_idx] = 1; + skip_plan.skipped_page_compressed_sizes[page_idx] = + offset_index.page_locations[page_idx].compressed_page_size; + append_row_range(range, &skip_plan.skipped_ranges); + } + if (!skip_plan.empty()) { + page_skip_plans->emplace(skip_plan.leaf_column_id, std::move(skip_plan)); + } + } + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_page_rows += row_group_rows - count_range_rows(*selected_ranges); + } + return Status::OK(); +} + } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 3c918fd650abbd..737fc4b58ae372 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -15,11 +15,14 @@ #pragma once +#include + #include #include #include #include #include +#include #include #include "common/status.h" @@ -118,6 +121,14 @@ struct ParquetColumnStatistics { bool has_any_statistics() const { return has_null_count || has_min_max; } }; +struct NativeParquetPageIndex { + tparquet::ColumnIndex column_index; + tparquet::OffsetIndex offset_index; +}; + +bool can_use_parquet_page_index(const format::FileScanRequest& request, + const RuntimeState* runtime_state); + // ============================================================================ // ============================================================================ // VExpr ZoneMap(TransformColumnStatistics + evaluate_zonemap_filter) @@ -160,4 +171,12 @@ Status select_row_group_ranges_by_page_index( ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr); +Status select_row_group_ranges_by_native_page_index( + const std::unordered_map& page_indexes, + const std::vector>& file_schema, + const format::FileScanRequest& request, int64_t row_group_rows, + std::vector* selected_ranges, std::map* page_skip_plans, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, + const RuntimeState* runtime_state = nullptr); + } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp index ee4955364110d5..a3f316c5e8558f 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -23,8 +23,8 @@ #include "common/config.h" #include "format/parquet/schema_desc.h" -#include "format/parquet/vparquet_file_metadata.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/reader/native/level_reader.h" #include "runtime/runtime_profile.h" @@ -97,7 +97,7 @@ CountColumnReader::~CountColumnReader() { sync_profile(); } -Status CountColumnReader::create(io::FileReaderSPtr file, const FileMetaData* metadata, +Status CountColumnReader::create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, const ParquetColumnSchema& root_schema, const format::LocalColumnIndex* projection, io::IOContext* io_ctx, bool enable_page_cache, const std::string& page_cache_file_key, diff --git a/be/src/format_v2/parquet/reader/count_column_reader.h b/be/src/format_v2/parquet/reader/count_column_reader.h index cd8ecb09473a53..789da8534d2ce3 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.h +++ b/be/src/format_v2/parquet/reader/count_column_reader.h @@ -27,13 +27,13 @@ #include "io/fs/file_reader_writer_fwd.h" namespace doris { -class FileMetaData; namespace io { struct IOContext; } } // namespace doris namespace doris::format::parquet { +class NativeParquetMetadata; struct ParquetColumnSchema; // Shape-only COUNT(nullable_col) reader. It uses v2's native LevelReader, so BYTE_ARRAY payloads // are skipped in the encoding stream and never copied into Arrow builders or Doris strings. @@ -41,8 +41,8 @@ class CountColumnReader { public: ~CountColumnReader(); - static Status create(io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, - const ParquetColumnSchema& root_schema, + static Status create(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, + int row_group_id, const ParquetColumnSchema& root_schema, const format::LocalColumnIndex* projection, io::IOContext* io_ctx, bool enable_page_cache, const std::string& page_cache_file_key, ParquetColumnReaderProfile profile, diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp index f3736afe892750..064bcffec8dd1f 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.cpp @@ -18,6 +18,7 @@ #include "format_v2/parquet/reader/native/byte_array_plain_decoder.h" #include +#include #include #include "core/column/column.h" @@ -38,25 +39,44 @@ Status read_length(const Slice* data, size_t* offset, uint32_t* length) { Status ByteArrayPlainDecoder::decode_binary_values(size_t num_values, ParquetBinaryValueConsumer& consumer) { - _binary_values.clear(); - _binary_values.reserve(num_values); + _payload_offsets.clear(); + _payload_offsets.reserve(num_values); + _value_offsets.clear(); + _value_offsets.reserve(num_values + 1); + _value_offsets.push_back(0); for (size_t row = 0; row < num_values; ++row) { uint32_t length = 0; RETURN_IF_ERROR(read_length(_data, &_offset, &length)); if (UNLIKELY(_offset > _data->size || length > _data->size - _offset)) { return Status::IOError("Can't read enough bytes in Parquet plain decoder"); } - _binary_values.emplace_back(_data->data + _offset, length); + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY batch exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); _offset += length; } - return consumer.consume(_binary_values.data(), _binary_values.size()); + _value_spans.clear(); + if (num_values != 0) { + _value_spans.push_back({.first = 0, .count = num_values}); + } + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), num_values, _value_spans); } Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelection& selection, ParquetBinaryValueConsumer& consumer) { - _binary_values.clear(); - _binary_values.reserve(selection.selected_values); + _payload_offsets.clear(); + _payload_offsets.reserve(selection.selected_values); + _value_offsets.clear(); + _value_offsets.reserve(selection.selected_values + 1); + _value_offsets.push_back(0); + _value_spans.clear(); + _value_spans.reserve(selection.ranges.size()); size_t range_index = 0; + size_t selected_in_range = 0; for (size_t row = 0; row < selection.total_values; ++row) { uint32_t length = 0; RETURN_IF_ERROR(read_length(_data, &_offset, &length)); @@ -65,18 +85,35 @@ Status ByteArrayPlainDecoder::decode_selected_binary_values(const ParquetSelecti } while (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first + selection.ranges[range_index].count) { + if (selected_in_range != 0) { + _value_spans.push_back({.first = _payload_offsets.size() - selected_in_range, + .count = selected_in_range}); + selected_in_range = 0; + } ++range_index; } if (range_index < selection.ranges.size() && row >= selection.ranges[range_index].first) { - _binary_values.emplace_back(_data->data + _offset, length); + if (UNLIKELY(_offset > std::numeric_limits::max() || + length > std::numeric_limits::max() - _value_offsets.back())) { + return Status::IOError("Parquet plain BYTE_ARRAY selection exceeds uint32 offsets"); + } + _payload_offsets.push_back(static_cast(_offset)); + _value_offsets.push_back(_value_offsets.back() + length); + ++selected_in_range; } _offset += length; } - DORIS_CHECK_EQ(_binary_values.size(), selection.selected_values); - if (_binary_values.empty()) { + if (selected_in_range != 0) { + _value_spans.push_back( + {.first = _payload_offsets.size() - selected_in_range, .count = selected_in_range}); + } + DORIS_CHECK_EQ(_payload_offsets.size(), selection.selected_values); + if (_payload_offsets.empty()) { return Status::OK(); } - return consumer.consume(_binary_values.data(), _binary_values.size()); + return consumer.consume_plain_byte_array(_data->data, _payload_offsets.data(), + _value_offsets.data(), _payload_offsets.size(), + _value_spans); } Status ByteArrayPlainDecoder::skip_values(size_t num_values) { diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h index 5bbb21a9660511..04f511a02f0d88 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -53,17 +53,23 @@ class ByteArrayPlainDecoder final : public Decoder { Status skip_values(size_t num_values) override; void release_scratch(size_t max_retained_bytes) override { - release_vector_if_oversized(&_binary_values, max_retained_bytes); + release_vector_if_oversized(&_payload_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_offsets, max_retained_bytes); + release_vector_if_oversized(&_value_spans, max_retained_bytes); } size_t retained_scratch_bytes() const override { - return _binary_values.capacity() * sizeof(StringRef); + return (_payload_offsets.capacity() + _value_offsets.capacity()) * sizeof(uint32_t) + + _value_spans.capacity() * sizeof(ParquetSelectionRange); } size_t active_scratch_bytes() const override { - return _binary_values.size() * sizeof(StringRef); + return (_payload_offsets.size() + _value_offsets.size()) * sizeof(uint32_t) + + _value_spans.size() * sizeof(ParquetSelectionRange); } private: - std::vector _binary_values; + std::vector _payload_offsets; + std::vector _value_offsets; + std::vector _value_spans; }; } // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 9537aa03d4dda3..92f00e63cbeba0 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -412,6 +412,15 @@ Status ColumnChunkReader::parse_page_header() { } int32_t page_num_values = _page_reader->is_header_v2() ? header->data_page_header_v2.num_values : header->data_page_header.num_values; + if (page_num_values < 0 || page_num_values > _metadata.num_values || + (!OFFSET_INDEX && + static_cast(page_num_values) > + static_cast(_metadata.num_values) - _chunk_parsed_values)) { + // Page counts are untrusted and feed both level decoders and scratch sizing. Bound each + // page by the column metadata before converting to unsigned counters. + return Status::Corruption("Parquet data page value count {} exceeds column total {}", + page_num_values, _metadata.num_values); + } _remaining_rep_nums = page_num_values; _remaining_def_nums = page_num_values; _remaining_num_values = page_num_values; @@ -1024,7 +1033,13 @@ Status ColumnChunkReader::load_page_nested_rows( } *cross_page = false; *result_rows = 0; - rep_levels.reserve(rep_levels.size() + _remaining_rep_nums); + // Reserve only the requested row frontier. One nested row may legitimately contain more + // values and grow the vector incrementally, but a forged page count must not allocate gigabytes + // before the level stream proves those values exist. + const size_t requested_frontier = + max_rows == std::numeric_limits::max() ? max_rows : max_rows + 1; + rep_levels.reserve(rep_levels.size() + + std::min(_remaining_rep_nums, requested_frontier)); while (_remaining_rep_nums) { level_t rep_level = _rep_level_get_next(); if (UNLIKELY(rep_level < 0)) { diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp index 77a99deea172e5..e563d0c03ff0a8 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -94,6 +94,9 @@ size_t LevelDecoder::get_levels(level_t* levels, size_t n) { n = std::min((size_t)_num_levels, n); size_t num_decoded = 0; if (_has_buffered_level && n > 0) { + if (!accept_level(_buffered_level)) { + return 0; + } levels[num_decoded++] = _buffered_level; _has_buffered_level = false; } @@ -103,7 +106,11 @@ size_t LevelDecoder::get_levels(level_t* levels, size_t n) { const size_t batch_decoded = _rle_decoder.GetBatch(_rle_scratch.data(), cast_set(remaining)); for (size_t i = 0; i < batch_decoded; ++i) { - levels[num_decoded + i] = cast_set(_rle_scratch[i]); + const level_t level = cast_set(_rle_scratch[i]); + if (!accept_level(level)) { + return 0; + } + levels[num_decoded + i] = level; } num_decoded += batch_decoded; } @@ -114,9 +121,14 @@ size_t LevelDecoder::get_levels(level_t* levels, size_t n) { n = std::min((size_t)_num_levels, n); size_t decoded = 0; for (; decoded < n; ++decoded) { - if (!_bit_packed_decoder.GetValue(_bit_width, &levels[decoded])) { + level_t level = -1; + if (!_bit_packed_decoder.GetValue(_bit_width, &level)) { break; } + if (!accept_level(level)) { + return 0; + } + levels[decoded] = level; } _num_levels -= decoded; return decoded; @@ -136,6 +148,9 @@ size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { if (_has_buffered_level) { *val = _buffered_level; _has_buffered_level = false; + if (!accept_level(*val)) { + return 0; + } decoded = 1; } else { uint16_t first = 0; @@ -143,12 +158,16 @@ size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { return 0; } *val = cast_set(first); + if (!accept_level(*val)) { + return 0; + } decoded = 1; } while (decoded < max_run) { const int32_t repeats = _rle_decoder.NextNumRepeats(); if (repeats > 0) { const level_t repeated = cast_set(_rle_decoder.GetRepeatedValue(0)); + if (!accept_level(repeated)) return 0; if (repeated != *val) break; const int32_t consume = std::min(repeats, cast_set(max_run - decoded)); _rle_decoder.GetRepeatedValue(consume); @@ -159,6 +178,7 @@ size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { uint16_t literal = 0; if (!_rle_decoder.GetLiteralValues(1, &literal)) break; const level_t next = cast_set(literal); + if (!accept_level(next)) return 0; if (next != *val) { // Batch RLE has no physical rewind; retain the one-value lookahead logically. _buffered_level = next; @@ -175,12 +195,16 @@ size_t LevelDecoder::get_next_run(level_t* val, size_t max_run) { !_bit_packed_decoder.GetValue(_bit_width, val)) { return 0; } + if (!accept_level(*val)) { + return 0; + } size_t decoded = 1; while (decoded < max_run) { level_t next = -1; if (!_bit_packed_decoder.GetValue(_bit_width, &next)) { break; } + if (!accept_level(next)) return 0; if (next != *val) { // The lookahead belongs to the following run, so cursor APIs must leave it unread. _bit_packed_decoder.Rewind(_bit_width); @@ -214,6 +238,9 @@ level_t LevelDecoder::get_next() { if (!decoded) { return -1; } + if (!accept_level(next)) { + return -1; + } --_num_levels; _last_level = next; _can_rewind = true; diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h index ee389a07c13a54..e232fe3a7be022 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.h +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -59,6 +59,18 @@ class LevelDecoder { size_t active_scratch_bytes() const { return _rle_scratch.size() * sizeof(uint16_t); } private: + bool accept_level(level_t level) { + if (level >= 0 && level <= _max_level) { + return true; + } + // Bit width rounds up to a power of two, so encoded values can fit the bit stream while + // still exceeding the schema maximum. Poison the decoder before any caller can use them. + _num_levels = 0; + _has_buffered_level = false; + _can_rewind = false; + return false; + } + tparquet::Encoding::type _encoding; level_t _bit_width = 0; level_t _max_level = 0; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index d4f8510f73021e..e3ed1895136c6f 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -35,9 +35,9 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_struct.h" -#include "format/parquet/vparquet_file_metadata.h" #include "format_v2/column_data.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" #include "runtime/runtime_state.h" namespace doris::format::parquet { @@ -155,7 +155,7 @@ NativeColumnReader::~NativeColumnReader() { Status NativeColumnReader::create( const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, - io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, @@ -204,8 +204,8 @@ Status NativeColumnReader::create( } Status NativeColumnReader::init( - io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, FieldSchema* field, - std::shared_ptr schema_node, + io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, + FieldSchema* field, std::shared_ptr schema_node, std::set projected_column_ids, const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, @@ -308,7 +308,13 @@ Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_ if (_nested && _profile.nested_batches != nullptr) { COUNTER_UPDATE(_profile.nested_batches, 1); } - _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + // Retained-capacity inspection walks the native reader tree. Check it periodically instead of + // on every small batch; row-group destruction is still the hard lifetime bound for scratch. + constexpr size_t SCRATCH_CHECK_BATCH_INTERVAL = 16; + if (++_batches_since_scratch_check >= SCRATCH_CHECK_BATCH_INTERVAL) { + _native_reader->release_batch_scratch(MAX_RETAINED_BATCH_SCRATCH_BYTES); + _batches_since_scratch_check = 0; + } if (*rows_read != rows) { return Status::Corruption("Native parquet reader returned {} rows, expected {} for {}", *rows_read, rows, _name); @@ -381,11 +387,11 @@ Status NativeColumnReader::skip(int64_t rows) { remaining -= gap; continue; } - const int64_t selected_rows = - std::min(remaining, range.start + range.length - _logical_row_position); + const int64_t selected_rows = detail::bounded_native_lazy_skip_rows( + std::min(remaining, range.start + range.length - _logical_row_position)); _skip_column->clear(); - // resize() preserves survivor bytes from the previous select(). An all-filtered nested read - // consumes the raw bitmap, so every slot must be explicitly reset before a lazy skip. + // Pending skips can span many filtered batches and are replayed for every lazy column. + // Chunking here bounds each dense bitmap while preserving one logical scheduler skip. _filter_scratch.assign(static_cast(selected_rows), 0); int64_t rows_read = 0; RETURN_IF_ERROR(read_with_filter(selected_rows, _filter_scratch.data(), true, _skip_column, diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index f561620719a67f..8dcd3590808f13 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -19,7 +19,9 @@ #include +#include #include +#include #include #include #include @@ -33,7 +35,6 @@ #include "format_v2/parquet/reader/native/column_reader.h" namespace doris { -class FileMetaData; class RuntimeState; namespace io { struct IOContext; @@ -42,6 +43,16 @@ struct IOContext; namespace doris::format::parquet { +class NativeParquetMetadata; + +namespace detail { +inline constexpr int64_t MAX_NATIVE_LAZY_SKIP_ROWS = std::numeric_limits::max(); + +inline int64_t bounded_native_lazy_skip_rows(int64_t rows) { + return std::min(rows, MAX_NATIVE_LAZY_SKIP_ROWS); +} +} // namespace detail + // Production adapter from FileScannerV2's selection-oriented reader contract to Doris' native // Parquet page/encoding reader. The owned native reader decodes page bytes directly into the final // Doris column. It never creates an Arrow Array/Builder, DecodedColumnView, or intermediate nested @@ -58,7 +69,7 @@ class NativeColumnReader final : public ParquetColumnReader { public: static Status create(const ParquetColumnSchema& column_schema, const format::LocalColumnIndex* projection, io::FileReaderSPtr file, - const FileMetaData* metadata, int row_group_id, + const NativeParquetMetadata* metadata, int row_group_id, const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, @@ -85,7 +96,7 @@ class NativeColumnReader final : public ParquetColumnReader { NativeColumnReader(const ParquetColumnSchema& schema, DataTypePtr projected_type, ParquetColumnReaderProfile profile); - Status init(io::FileReaderSPtr file, const FileMetaData* metadata, int row_group_id, + Status init(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, FieldSchema* field, std::shared_ptr schema_node, std::set projected_column_ids, const std::vector& selected_ranges, @@ -121,6 +132,7 @@ class NativeColumnReader final : public ParquetColumnReader { // FileScannerV2 batch contributes only its delta to RuntimeProfile. native::ColumnReader::ColumnStatistics _reported_native_stats; std::vector _filter_scratch; + size_t _batches_since_scratch_check = 0; MutableColumnPtr _skip_column; MutableColumnPtr _dictionary_id_column; MutableColumnPtr _matched_dictionary_ids; diff --git a/be/src/io/fs/file_meta_cache.cpp b/be/src/io/fs/file_meta_cache.cpp index f7d98461035afb..f97b2f80cd6ee6 100644 --- a/be/src/io/fs/file_meta_cache.cpp +++ b/be/src/io/fs/file_meta_cache.cpp @@ -40,15 +40,4 @@ std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, _file_description.file_size == -1 ? file_reader->size() : _file_description.file_size); } -std::string FileMetaCache::get_key(io::FileReaderSPtr file_reader, - const io::FileDescription& file_description, - bool enable_mapping_varbinary, - bool enable_mapping_timestamp_tz) { - auto key = get_key(std::move(file_reader), file_description); - // Schema mapping changes the cached object, so those options are part of its identity. - key.push_back(static_cast(enable_mapping_varbinary)); - key.push_back(static_cast(enable_mapping_timestamp_tz)); - return key; -} - } // namespace doris diff --git a/be/src/io/fs/file_meta_cache.h b/be/src/io/fs/file_meta_cache.h index 054914a9519041..0c62c963ce122d 100644 --- a/be/src/io/fs/file_meta_cache.h +++ b/be/src/io/fs/file_meta_cache.h @@ -41,10 +41,6 @@ class FileMetaCache { static std::string get_key(io::FileReaderSPtr file_reader, const io::FileDescription& _file_description); - static std::string get_key(io::FileReaderSPtr file_reader, - const io::FileDescription& file_description, - bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); - bool lookup(const std::string& key, ObjLRUCache::CacheHandle* handle) { return _cache.lookup({key}, handle); } diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp index 1dec32bf8acebc..3ced956af03078 100644 --- a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -73,6 +73,13 @@ class TestParquetDecodeSource final : public ParquetDecodeSource { ++_dictionary_generation; } + template + void set_fixed_dictionary(const std::vector& values, std::vector indices) { + std::vector encoded(values.size() * sizeof(T)); + memcpy(encoded.data(), values.data(), encoded.size()); + set_dictionary(std::move(encoded), sizeof(T), std::move(indices)); + } + Status decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) override { DORIS_CHECK_LE((_fixed_offset + num_values) * _value_width, _fixed_values.size()); const uint8_t* begin = _fixed_values.data() + _fixed_offset * _value_width; @@ -189,6 +196,138 @@ TEST(DataTypeSerDeParquetTest, RescalesFixedBinaryDecimalDirectly) { EXPECT_EQ(data[1].value, -12340); } +TEST(DataTypeSerDeParquetTest, RescalesExactIntegerDecimalsWithoutRowFailures) { + TestParquetDecodeSource source; + source.set_fixed_values({12300, -98700, 2147483600}); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 10, + .decimal_scale = 4}; + ParquetMaterializationState state; + DataTypeDecimal64 type(9, 2); + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*column, source, context, 3, state).ok()); + const auto& data = assert_cast(*column).get_data(); + // Exact scale reduction can stay on the fast integer path without per-row failure bookkeeping. + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, -987); + EXPECT_EQ(data[2].value, 21474836); +} + +TEST(DataTypeSerDeParquetTest, DecimalScaleDownRejectsLossyPlainAndDictionaryValues) { + ParquetDecodeContext plain_context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 6, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + { + TestParquetDecodeSource source; + source.set_fixed_values({1230, 1234}); + IColumn::Filter null_map(2, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, 123); + EXPECT_EQ(data[1].value, 0); + EXPECT_EQ(null_map, IColumn::Filter({0, 1})); + } + { + TestParquetDecodeSource source; + source.set_fixed_values({1234}); + ParquetMaterializationState state; + state.enable_strict_mode = true; + auto column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*column, source, plain_context, 1, state) + .ok()); + EXPECT_EQ(column->size(), 0); + } + { + TestParquetDecodeSource source; + source.set_fixed_dictionary({1230, 1234}, {1, 0, 1}); + auto dictionary_context = plain_context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + IColumn::Filter null_map(3, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE( + type.get_serde() + ->read_column_from_parquet(*column, source, dictionary_context, 3, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[1].value, 123); + EXPECT_EQ(null_map, IColumn::Filter({1, 0, 1})); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalUsesWideIntermediateBeforeNarrowing) { + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 19, + .decimal_scale = 0}; + DataTypeDecimal32 type(9, 0); + constexpr int64_t WRAPS_TO_ONE_IN_INT32 = 4294967297LL; + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_fixed_dictionary({WRAPS_TO_ONE_IN_INT32}, {0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_values({WRAPS_TO_ONE_IN_INT32}); + context.encoding = ParquetValueEncoding::PLAIN; + } + IColumn::Filter null_map(1, 0); + ParquetMaterializationState state; + state.conversion_failure_null_map = &null_map; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 1, state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1})); + EXPECT_EQ(assert_cast(*column).get_data()[0].value, 0); + } +} + +TEST(DataTypeSerDeParquetTest, DecimalAcceptsSignExtendedBinaryAfterWideExactScaling) { + const std::vector values {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xCE, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFB, 0x32}; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::FIXED_LEN_BYTE_ARRAY, + .logical_type = ParquetLogicalType::DECIMAL, + .type_length = 8, + .decimal_precision = 19, + .decimal_scale = 2}; + DataTypeDecimal32 type(5, 1); + for (bool dictionary : {false, true}) { + TestParquetDecodeSource source; + if (dictionary) { + source.set_dictionary(values, 8, {1, 0}); + context.encoding = ParquetValueEncoding::DICTIONARY; + } else { + source.set_fixed_bytes(values, 8); + context.encoding = ParquetValueEncoding::PLAIN; + } + ParquetMaterializationState state; + auto column = type.create_column(); + + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0].value, dictionary ? -123 : 123); + EXPECT_EQ(data[1].value, dictionary ? 123 : -123); + } +} + TEST(DataTypeSerDeParquetTest, ReusesTypedDictionary) { TestParquetDecodeSource source; std::vector dictionary {'a', 'a', 'b', 'b'}; diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index 9aa793e3a04933..fc5385dd56972a 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -108,13 +108,6 @@ TEST(FileMetaCacheTest, KeyGenerationFromFileReader) { std::string key2 = FileMetaCache::get_key(reader2, desc2); std::string expected_key2 = FileMetaCache::get_key(file_name, 0, 300); EXPECT_EQ(key2, expected_key2); - - const std::string default_mapping = FileMetaCache::get_key(reader2, desc2, false, false); - const std::string varbinary_mapping = FileMetaCache::get_key(reader2, desc2, true, false); - const std::string timestamp_mapping = FileMetaCache::get_key(reader2, desc2, false, true); - EXPECT_NE(default_mapping, varbinary_mapping); - EXPECT_NE(default_mapping, timestamp_mapping); - EXPECT_NE(varbinary_mapping, timestamp_mapping); } TEST(FileMetaCacheTest, KeyContentVerification) { std::string file_name = "/path/to/file"; diff --git a/be/test/format/parquet/parquet_thrift_test.cpp b/be/test/format/parquet/parquet_thrift_test.cpp index d9609648fe432a..0fe101db598138 100644 --- a/be/test/format/parquet/parquet_thrift_test.cpp +++ b/be/test/format/parquet/parquet_thrift_test.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include @@ -97,23 +96,6 @@ TEST_F(ParquetThriftReaderTest, normal) { } } -TEST_F(ParquetThriftReaderTest, ReusesArrowMetadataAdapterForCachedNativeFooter) { - auto local_fs = io::global_local_filesystem(); - io::FileReaderSPtr reader; - ASSERT_TRUE(local_fs->open_file("./be/test/exec/test_data/parquet_scanner/localfile.parquet", - &reader) - .ok()); - std::unique_ptr metadata; - size_t meta_size = 0; - ASSERT_TRUE(parse_thrift_footer(reader, &metadata, &meta_size, nullptr, true, true).ok()); - - std::shared_ptr<::parquet::FileMetaData> first; - std::shared_ptr<::parquet::FileMetaData> second; - ASSERT_TRUE(metadata->get_arrow_metadata(&first).ok()); - ASSERT_TRUE(metadata->get_arrow_metadata(&second).ok()); - EXPECT_EQ(first, second); -} - TEST_F(ParquetThriftReaderTest, complex_nested_file) { // hive-complex.parquet is the part of following table: // complex_nested_table( diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 5fa45d77b223d3..c823abb2812702 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -64,6 +64,31 @@ class CaptureBinaryConsumer final : public ParquetBinaryValueConsumer { std::vector refs; }; +class CapturePlainBinaryLayoutConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + legacy_consume_called = true; + return Status::InternalError("PLAIN BYTE_ARRAY used the legacy StringRef path"); + } + + Status consume_plain_byte_array( + const char* encoded_data, const uint32_t* payload_offsets, + const uint32_t* value_offsets, size_t num_values, + const std::vector& value_spans) override { + base = encoded_data; + source_offsets.assign(payload_offsets, payload_offsets + num_values); + output_offsets.assign(value_offsets, value_offsets + num_values + 1); + spans = value_spans; + return Status::OK(); + } + + const char* base = nullptr; + bool legacy_consume_called = false; + std::vector source_offsets; + std::vector output_offsets; + std::vector spans; +}; + class CaptureFixedConsumer final : public ParquetFixedValueConsumer { public: Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { @@ -248,6 +273,32 @@ Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, + int64_t metadata_values = std::numeric_limits::max(), + level_t max_repetition_level = 1) { + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(metadata_values); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + FieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = max_repetition_level; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + std::vector rep_levels; + size_t result_rows = 0; + bool cross_page = false; + return chunk_reader.load_page_nested_rows(rep_levels, 1, &result_rows, &cross_page); +} + TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { int32_t dictionary_length = 0; auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); @@ -403,6 +454,41 @@ TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCurso EXPECT_EQ(trailing_rle_boolean.values(), std::vector({1})); } +TEST(ParquetV2NativeDecoderTest, PlainByteArrayPublishesOffsetsAndCoalescedSelectionSpans) { + const std::vector strings {"zero", "one", "two", "three", + "four", "five", "six", "seven"}; + auto encoded = encode_plain_byte_arrays(strings); + Slice slice(encoded.data(), encoded.size()); + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, decoder) + .ok()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + + const ParquetSelection selection { + .total_values = 7, + .selected_values = 4, + .ranges = {{.first = 1, .count = 2}, {.first = 5, .count = 2}}}; + CapturePlainBinaryLayoutConsumer consumer; + ASSERT_TRUE(decoder->decode_selected_binary_values(selection, consumer).ok()); + EXPECT_FALSE(consumer.legacy_consume_called); + EXPECT_EQ(consumer.output_offsets, std::vector({0, 3, 6, 10, 13})); + ASSERT_EQ(consumer.source_offsets.size(), selection.selected_values); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[0], 3), "one"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[1], 3), "two"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[2], 4), "five"); + EXPECT_EQ(std::string_view(consumer.base + consumer.source_offsets[3], 3), "six"); + ASSERT_EQ(consumer.spans.size(), 2); + EXPECT_EQ(consumer.spans[0].first, 0); + EXPECT_EQ(consumer.spans[0].count, 2); + EXPECT_EQ(consumer.spans[1].first, 2); + EXPECT_EQ(consumer.spans[1].count, 2); + + CaptureBinaryConsumer trailing; + ASSERT_TRUE(decoder->decode_binary_values(1, trailing).ok()); + ASSERT_EQ(trailing.refs.size(), 1); + EXPECT_EQ(trailing.refs[0].to_string_view(), "seven"); +} + TEST(ParquetV2NativeDecoderTest, SparsePlainFixedDecodeDoesNotRetainGatherBuffer) { constexpr size_t value_count = 1UL << 18; std::vector integers(value_count); @@ -661,6 +747,55 @@ TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); } +TEST(ParquetV2NativeDecoderTest, RejectsLevelsAboveSchemaMaximumOnEveryDecodePath) { + auto make_decoder = [](tparquet::Encoding::type encoding) { + static char encoded[] = {0x03}; // width=2 value=3, while the schema maximum is 2. + static char rle_encoded[] = {0x02, 0x00, 0x00, 0x00, 0x02, 0x03}; + Slice levels = encoding == tparquet::Encoding::BIT_PACKED + ? Slice(encoded, sizeof(encoded)) + : Slice(rle_encoded, sizeof(rle_encoded)); + LevelDecoder decoder; + EXPECT_TRUE(decoder.init(&levels, encoding, 2, 1).ok()); + return decoder; + }; + + for (auto encoding : {tparquet::Encoding::BIT_PACKED, tparquet::Encoding::RLE}) { + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_levels(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + level_t value = -1; + EXPECT_EQ(decoder.get_next_run(&value, 1), 0); + } + { + auto decoder = make_decoder(encoding); + EXPECT_EQ(decoder.get_next(), -1); + } + } +} + +TEST(ParquetV2NativeDecoderTest, NestedReadersRejectRleAndBitPackedLevelsAboveMaximum) { + for (auto encoding : {tparquet::Encoding::RLE, tparquet::Encoding::BIT_PACKED}) { + const std::vector payload = encoding == tparquet::Encoding::RLE + ? std::vector {2, 0, 0, 0, 2, 3} + : std::vector {3}; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(1); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + + EXPECT_TRUE(load_malformed_nested_page(header, payload, 1, 2).is()); + } +} + TEST(ParquetV2NativeDecoderTest, TruncatedBooleanStreamsFailWhileSkipping) { std::unique_ptr decoder; ASSERT_TRUE( @@ -1093,6 +1228,34 @@ TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption EXPECT_TRUE(parse_header(impossible_counts).is()); } +TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { + for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(4); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(std::numeric_limits::max()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(std::numeric_limits::max()); + header.data_page_header_v2.__set_num_rows(1); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } + // The four-byte V1 level length (or V2 value payload) contains no advertised levels. The + // reader must report corruption without reserving INT_MAX level slots first. + EXPECT_TRUE(load_malformed_nested_page(header, {0, 0, 0, 0}).is()); + } +} + TEST(ParquetV2NativeDecoderTest, PageDecompressionRejectsBothSizeMismatchDirections) { BlockCompressionCodec* codec = nullptr; ASSERT_TRUE(get_block_compression_codec(tparquet::CompressionCodec::SNAPPY, &codec).ok()); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 7c3007eb732b2f..7028832a66dc2e 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -51,6 +51,7 @@ #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_reader.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "gen_cpp/PlanNodes_types.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -344,6 +345,23 @@ void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group write_table(file_path, table, row_group_size, false, false, enable_statistics); } +void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) { + std::vector ids(rows); + std::vector first(rows); + std::vector second(rows); + std::iota(ids.begin(), ids.end(), 0); + for (size_t row = 0; row < rows; ++row) { + first[row] = static_cast(row + 10); + second[row] = static_cast(row + 20); + } + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false), + arrow::field("first", arrow::int32(), false), + arrow::field("second", arrow::int32(), false)}); + auto table = arrow::Table::Make( + schema, {build_int32_array(ids), build_int32_array(first), build_int32_array(second)}); + write_table(file_path, table, rows, false, false, false); +} + void write_binary_minmax_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("text", arrow::utf8(), false), @@ -538,6 +556,34 @@ TEST(ParquetScanAdaptivePredicateTest, OrdersByObservedCostPerRejectedRow) { EXPECT_EQ(prefetched, std::vector({1})); } +TEST(ParquetScanAdaptivePredicateTest, SamplesWarmupThenAtLowFrequency) { + using format::parquet::detail::should_sample_adaptive_predicate; + for (size_t samples = 0; samples < 8; ++samples) { + EXPECT_TRUE(should_sample_adaptive_predicate(samples, samples)); + } + EXPECT_FALSE(should_sample_adaptive_predicate(8, 9)); + EXPECT_TRUE(should_sample_adaptive_predicate(8, 16)); + EXPECT_FALSE(should_sample_adaptive_predicate(9, 17)); + EXPECT_TRUE(should_sample_adaptive_predicate(9, 32)); +} + +TEST(ParquetScanSmallFileTest, StagesOnlyBoundedHttpObjects) { + using format::parquet::detail::should_stage_small_http_file; + EXPECT_TRUE(should_stage_small_http_file("http://host/tiny.parquet", 512, 1024)); + EXPECT_TRUE(should_stage_small_http_file("https://host/tiny.parquet", 1024, 1024)); + EXPECT_FALSE(should_stage_small_http_file("https://host/large.parquet", 1025, 1024)); + EXPECT_FALSE(should_stage_small_http_file("/tmp/tiny.parquet", 512, 1024)); + EXPECT_FALSE(should_stage_small_http_file("s3://bucket/tiny.parquet", 512, 1024)); +} + +TEST(ParquetScanSelectionTest, NativeLazySkipBitmapIsBounded) { + using format::parquet::detail::MAX_NATIVE_LAZY_SKIP_ROWS; + using format::parquet::detail::bounded_native_lazy_skip_rows; + EXPECT_EQ(bounded_native_lazy_skip_rows(1), 1); + EXPECT_EQ(bounded_native_lazy_skip_rows(MAX_NATIVE_LAZY_SKIP_ROWS + 1), + MAX_NATIVE_LAZY_SKIP_ROWS); +} + TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { write_int_pair_parquet_file(_file_path, 2); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); @@ -628,7 +674,8 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater .first_file_row = ConditionCacheContext::GRANULE_SIZE, .row_group_rows = ConditionCacheContext::GRANULE_SIZE, .selected_ranges = {{.start = 0, .length = ConditionCacheContext::GRANULE_SIZE}}, - .page_skip_plans = {}}); + .page_skip_plans = {}, + .offset_indexes = {}}); format::parquet::ParquetScanScheduler scheduler; scheduler.set_plan(std::move(plan)); @@ -1162,7 +1209,8 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { while (!eof) { Block block = build_file_block(schema); size_t rows = 0; - ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; if (rows == 0) { continue; } @@ -1181,6 +1229,9 @@ TEST_F(ParquetScanTest, PredicateColumnsFilterRoundByRound) { EXPECT_EQ(counter_value(profile, "RawRowsRead"), 6); EXPECT_EQ(counter_value(profile, "SelectedRows"), 2); EXPECT_EQ(counter_value(profile, "RowsFilteredByConjunct"), 4); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 1); + EXPECT_GT(counter_value(profile, "PredicateCompactionBytes"), 0); + ASSERT_NE(profile.get_counter("PredicateCompactionTime"), nullptr); EXPECT_EQ(counter_value(profile, "ReaderReadRows"), 10); EXPECT_EQ(counter_value(profile, "ReaderSelectRows"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 2); @@ -1297,6 +1348,35 @@ TEST_F(ParquetScanTest, PendingLazySkipDoesNotCrossRowGroupReset) { EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); } +TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunks) { + constexpr size_t ROWS = 70000; + write_long_prefix_parquet_file(_file_path, ROWS); + auto reader = create_reader(); + reader->set_batch_size(1024); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(2)).ok()); + request->conjuncts.push_back( + create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, ROWS - 1)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_element(0), ROWS - 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_element(0), ROWS + 9); + EXPECT_EQ(int32_data_column(*block.get_by_position(2).column).get_element(0), ROWS + 19); +} + // Scenario: a nested lazy column stays behind while id=1 is rejected. Flushing skip(1) must consume // the complete repetition/definition-level span for the first list, then materialize the remaining // two parent rows without corrupting their child boundaries. diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index e86808bfda7e61..d41249dd8af90b 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -120,6 +120,9 @@ format-specific checklist when reviewing Parquet or ORC. - V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. +- Footer parsing, schema-ID assignment, retained serialized bytes, and the cached metadata payload + must also be v2-owned. Reusing a stable base file identity is allowed, but require a v2 cache type + discriminator so v1/v2 metadata objects can never be cross-cast. - Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, builders, `DecodedColumnView`, or another decoded leaf batch. @@ -171,6 +174,9 @@ format-specific checklist when reviewing Parquet or ORC. addition/multiplication for byte extents; bound BYTE_ARRAY dictionary entry counts and IDs before allocation/indexing; and require DELTA_BYTE_ARRAY prefixes to fit the previous reconstructed value. BOOLEAN RLE and DELTA skip paths must consume bounded chunks and fail on short streams. +- Validate every decoded definition/repetition level against the schema maximum in batch, run, and + single-value cursor APIs. Page value counts must not drive eager nested scratch allocation; + reserve from the requested parent-row frontier and cover tiny-payload huge-count Page V1/V2 files. - Before Snappy decompression, inspect the encoded uncompressed length and validate destination capacity. Page V1, Page V2, and dictionary pages must produce exactly the declared decoded size; an UNCOMPRESSED dictionary page must also declare equal compressed and uncompressed sizes. @@ -199,9 +205,16 @@ format-specific checklist when reviewing Parquet or ORC. DATETIME, TIME, and DECIMAL failures insert a default nested value and mark the corresponding NULL only in non-strict mode; strict or non-nullable reads return the error. Dictionary failures follow the selected dictionary IDs to output rows. +- Keep Parquet decimals in a source-width or wider intermediate until exact scaling, target + precision, and overflow checks succeed. Scale-down with a non-zero remainder is a conversion + failure; plain and dictionary integer/binary paths must narrow only afterward. - For cold small-file tests, separate footer I/O/Thrift parse from Arrow metadata adaptation. V2 may retain the already-read serialized footer to avoid serializing the same Thrift object again; v1 opens must not retain those bytes by default. +- For HTTP Parquet objects at or below `in_memory_file_size`, v2 stages the complete object from + byte zero before native page access. Verify both cold and footer-cache-hit scans: some Range + servers accept the capability probe but return HTTP 200 for a near-EOF overlong range, so warm + scans must not depend on the footer read having populated an incidental transport buffer. - Identical fixed-width POD values append with one bulk copy. FIXED_LEN_BYTE_ARRAY strings copy the dense byte span once and synthesize offsets; validate this execution contract without flaky wall-clock assertions. @@ -231,8 +244,28 @@ format-specific checklist when reviewing Parquet or ORC. - For safe staged single-column predicates, review the observed cost/rejection ordering and its cold-start behavior. Reordering is allowed only after every candidate has a sample; prefetch may stop at a low-probability reach prefix, while output-column prefetch may start early only after a - learned high survival ratio. Cache the batch SelectionVector's dense bitmap by generation so a - wide lazy projection does not rebuild the same O(batch) filter for every column. + learned high survival ratio. Static conjunct schedules and position maps belong to the scanner + lifecycle; after eight warm-up samples, per-predicate clocks should be sampled only periodically. + Cache the batch SelectionVector's dense bitmap by generation so a wide lazy projection does not + rebuild the same O(batch) filter for every column. +- Single-column predicate rounds may keep previously read columns in their original row mappings. + Require one alignment compaction before multi-column, delete, or output boundaries, and expose + its time, bytes, and count instead of charging repeated movement invisibly to predicate time. + Reused compaction masks must be fully cleared when coordinate-space sizes shrink. +- PLAIN BYTE_ARRAY must not parse lengths in both decoder and destination column. Review the direct + payload-offset/cumulative-offset contract, uint32 overflow checks, surviving-span coverage, and + the legacy consumer fallback for non-string logical types. +- Production PageIndex planning must consume native Compact Thrift ColumnIndex/OffsetIndex objects. + Coalesce adjacent serialized index ranges and transfer validated OffsetIndexes into execution so + the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; any remaining + Arrow metadata adapter must expose its time and retained bytes until it is removed. +- Fixed-width conversion fast paths may remove row branches only when the source domain provably + fits the target domain. Narrowing, timestamp, decimal scaling, strict rollback, and non-strict + NULL marking must retain corrupt-value tests. +- Recursive reader-profile publication and retained-scratch inspection should be amortized, but Row + Group reset/EOF/close must force the final profile delta before reader destruction. +- Accumulated lazy-column skips must not allocate one dense byte per rejected prefix row. Require a + fixed chunk bound (including multiple lazy columns) or a level-only/all-filtered cursor contract. - Register Parquet Page Cache ranges only for surviving projected Column Chunks, require a stable file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 5970381eab73d6..d21d4688f2fb00 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -82,10 +82,11 @@ the production v2 path never instantiates the v1 `ParquetColumnReader`: | Compatibility removal | Remove Arrow data-read adapters after type/encoding/page/writer compatibility and performance gates pass. Production v2 never falls back from a selected native reader to Arrow; an unsupported combination returns an explicit error. | Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or -builders. Arrow remains in footer/schema planning and dictionary/statistics probing. It is not a -runtime fallback: after a column selects the native reader, decode errors are returned directly. -The production target eventually removes the remaining Arrow metadata objects; tests may also use -Arrow as a fixture writer or oracle. +builders. V2 parses and owns the Thrift footer and physical schema; a lazy Arrow metadata adapter +remains only for planner consumers that have not yet migrated. It is not a runtime fallback: after +a column selects the native reader, decode errors are returned directly. The production target +eventually removes that remaining planner adapter; tests may also use Arrow as a fixture writer or +oracle. All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness and performance control. Compatibility is demonstrated through differential tests and the explicit @@ -379,7 +380,8 @@ but moves sparse range traversal inside the concrete decoder instead of repeated SerDe consumer for every selected run. The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk -copies; PLAIN BYTE_ARRAY scans every length once but records only selected references; dictionary +copies; PLAIN BYTE_ARRAY scans every length once and publishes compact source offsets, cumulative +output offsets, and coalesced surviving spans directly to the SerDe; dictionary encoding decodes and validates the complete ID batch before gathering selected IDs; BOOLEAN, DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate @@ -400,7 +402,7 @@ that parsing step both feed the same level and value-decoder contracts. | Encoding family | Native responsibility | | --- | --- | -| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and sparse variable strings scan lengths once | +| PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and variable strings reuse decoder-produced offsets without a `StringRef[]` staging pass | | RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, decode and validate the complete ID batch, then gather selected IDs | | RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | | DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | @@ -421,6 +423,12 @@ the preceding reconstructed value and checks the reconstructed and aggregate byt RLE and DELTA skip paths operate in bounded chunks and reject short streams instead of advancing a partially consumed cursor as if the request succeeded. +Level bit width is only a storage bound: for a schema maximum of 2, two bits can still encode the +invalid value 3. Batch, run, and single-value level cursors therefore reject every decoded value +above the schema maximum. Nested page traversal reserves only the requested parent-row frontier; +an untrusted Page V1/V2 `num_values` cannot trigger an eager page-sized allocation before the level +payload proves those entries exist. + Compression has an exact-size contract. Snappy's encoded uncompressed length is checked against the destination capacity before decompression; Page V1, Page V2, and dictionary decode must then produce exactly the size declared in the page header. For the UNCOMPRESSED codec, dictionary compressed and @@ -440,9 +448,10 @@ that row monotonicity alone cannot detect; otherwise sequential traversal preser Encoding decoders expose contiguous physical spans and advance encoded-stream cursors. The selected `DataTypeSerDe` consumes those spans and writes directly into the Doris mutable column. Fixed-width -types append contiguous runs; string-like paths gather `StringRef` values in persistent decoder -scratch before a batched append. References remain valid only while the page or dictionary buffer -is pinned by the persistent leaf reader. +types append contiguous runs. PLAIN BYTE_ARRAY publishes payload offsets, cumulative destination +offsets, and coalesced survivor spans so string columns perform larger range copies without a +`StringRef[]` staging array. Other binary encodings may use persistent references that remain valid +only while the page or dictionary buffer is pinned by the persistent leaf reader. The direct path covers identical logical types, string-family compatibility, decimal precision/scale changes, integer changes, and FLOAT-to-DOUBLE widening. Less common table-schema @@ -460,7 +469,10 @@ physical/logical conversion while appending to the final column. Decimal and FIXED_LEN_BYTE_ARRAY direct paths validate the physical byte width, decode big-endian two's-complement values with correct sign extension, and apply precision/scale conversion exactly -once. Date, timestamp, INT96, unsigned annotations, CHAR/VARCHAR, and timezone conversions retain +once. Decimal integer and binary inputs stay in a 256-bit intermediate through exact scale-down, +scale-up overflow, and target-precision checks; discarded non-zero digits are conversion failures, +and narrowing happens only after success. Date, timestamp, INT96, unsigned annotations, +CHAR/VARCHAR, and timezone conversions retain the same semantic checks as the general conversion path. A fast path is enabled only when those checks prove the result is equivalent. In particular, legacy converted `TIMESTAMP_MILLIS` and `TIMESTAMP_MICROS` are UTC-adjusted, while an unannotated INT64 timestamp has distinct @@ -513,9 +525,11 @@ key/value repetition vectors are intentionally not compared because a nested MAP additional repetition levels. STRUCT siblings similarly normalize repetition to the current parent boundary; deeper child collection repetition cannot redefine or invalidate the sibling shape. -Level scratch is sized by decoded level entries, not by the 16-bit parent batch cap. Long repeated -rows and long null/non-null runs are split into representable internal runs without introducing a -new row boundary. Tests cover null ancestors, empty collections, null elements/values, nested +Level scratch grows with level entries that were actually decoded, while its initial reservation is +bounded by the requested parent-row frontier rather than the page header's untrusted value count. +Long repeated rows can still grow incrementally beyond that frontier, and long null/non-null runs +are split into representable internal runs without introducing a new row boundary. Tests cover null +ancestors, empty collections, null elements/values, nested STRUCT-in-ARRAY and ARRAY-in-STRUCT shapes, sibling page misalignment, and rows spanning pages and batches. @@ -643,7 +657,8 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | -| Footer metadata cache | Immutable parsed native metadata and, for a v2 cold miss, the serialized footer bytes already fetched from storage | Stable file identity matching v1: path plus size and trustworthy modification/version information, with schema-affecting options in the parsed-object key | Avoid repeated footer I/O, Thrift parsing, schema construction, and v2's former Thrift re-serialization before Arrow metadata adaptation | +| Footer metadata cache | V2-owned immutable Thrift metadata, v2 physical schema, and the serialized footer bytes already fetched from storage | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, unsafe cross-version cache casts, and Thrift re-serialization before Arrow metadata adaptation | +| Small HTTP object staging | Complete object bytes for files at or below `in_memory_file_size` | Per-reader v2 wrapper; loaded once from byte zero and released with the file context | Collapse cold small-file requests and keep footer-cache-hit scans independent of server-specific near-EOF Range behavior | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | | Condition Cache | Condition-surviving granule bitmap | Managed by condition and file-range context | Reuse filtering results before reading columns | @@ -651,19 +666,28 @@ flowchart TB ### Footer Cache Parity with V1 -V2 uses the same cacheability and invalidation policy as v1 rather than introducing a second notion -of file identity. A hit returns immutable metadata that can be shared by readers; mutable Row Group, -selection, decoder, and scratch state remains per reader. Path-only keys are insufficient. When a -trustworthy version identity is unavailable, the footer is read and parsed without publishing a -reusable entry. Parse failures, short files, encrypted/unsupported metadata, and schema-affecting -option changes cannot populate or reuse a successful entry. - -V2 calls the same footer parser/cache and key builder as v1. On either a cache hit or miss, the -immutable native Thrift metadata is the owner. On a v2 cold miss, the parser also retains the exact -serialized footer bytes already in memory, and the lazy Arrow planner adapter parses those bytes -directly instead of serializing the new Thrift object again. V1 keeps retention disabled by default. -The Arrow adapter is cached at the footer-object lifecycle; it is never the cache value and its -lifetime does not enter the native decoder. +V2 reuses the common stable file identity policy, but owns its footer parser, schema lifecycle, +metadata payload type, and cache-key suffix entirely under `format_v2`. A v2 cache entry can be +shared by v2 readers but can never be cast as a v1 metadata object. Mutable Row Group, selection, +decoder, and scratch state remains per reader. Path-only keys are insufficient. Parse failures, +short files, unsupported metadata, and schema-affecting option changes cannot populate or reuse a +successful entry. No change to the v1 parser or metadata cache behavior is required. + +On a v2 cold miss, the parser retains the exact serialized footer bytes already in memory, and the +lazy Arrow planner adapter parses those bytes directly instead of serializing the Thrift object +again. The adapter is cached inside the v2 footer-object lifecycle; it is never the cache payload +type and its lifetime does not enter the native decoder. Production ColumnIndex/OffsetIndex planning reads and +parses Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and the +validated OffsetIndex objects are transferred into Row Group execution instead of being fetched a +second time. The Arrow PageIndexReader path remains only as a test oracle. Until the remaining +row-group metadata planner adapter is removed, its construction cost and retained tree size are +visible as `ArrowMetadataAdapterTime/Bytes`. + +Before the footer lookup, v2 wraps bounded HTTP Parquet objects in an in-memory reader. The wrapper +loads from byte zero on its first physical access, whether that access is a cold footer read or a +warm data-page read after a footer-cache hit. This preserves identical cold/warm behavior for HTTP +servers that advertise Range support but answer an overlong near-EOF range with HTTP 200, and keeps +the compatibility policy out of the v1 reader. ### Page Cache Parity with V1 @@ -766,6 +790,11 @@ selection generation and batch shape, so every lazily materialized output reader bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary capacity remains reusable. +Fully rejected batches accumulate a logical lag for lazy columns. When a later batch survives, each +lazy reader consumes that lag in at most 65,535-row dense-filter chunks. This keeps the scheduler's +single logical skip while bounding adapter-owned bitmap capacity independently of prefix length and +lazy projection width. + ## 11. Correctness, Fallback, and Capability Boundaries V2 follows a prove-before-skip rule. Missing indexes, unsupported types, expressions that cannot be @@ -803,9 +832,11 @@ then I/O/cache health. Total ScanTime alone does not identify the cause. The visible timer hierarchy is `FileScannerV2 -> TableReader -> FileReader -> IO`; the format-specific `ParquetReader` subtree belongs to `FileReader`. Scanner lifecycle and Split work, table-semantic restoration, format metadata/index/decode/materialization, and physical I/O are -charged to their owning layer. Recursive native child-reader statistics are flushed at every batch -boundary, including empty-selection and early-return paths, so a query cannot be slow merely because -its counters are waiting for reader close. +charged to their owning layer. Native child-reader statistics accumulate in plain integers and are +published every 16 batches. Row Group reset, EOF, and reader close force the final delta before +destroying the reader tree, so short files retain complete attribution without paying recursive +profile publication on every tiny batch. Retained-scratch inspection uses the same amortized cadence +and a Row Group remains its hard lifetime bound. ```mermaid flowchart TD @@ -830,8 +861,9 @@ flowchart TD | Page index pruning | How many indexes were checked, pages/rows were pruned, ranges selected, and pages skipped? | | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | +| Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | | Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | -| Metadata lifecycle | How much time was spent reading the footer, parsing metadata, lazily materializing page indexes, and evaluating page-index predicates? | +| Metadata lifecycle | How much time was spent reading the footer, adapting remaining Arrow planner metadata (`ArrowMetadataAdapterTime/Bytes`), natively reading/parsing page indexes, and evaluating page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | From 0eaf04be139e0c8d170765bf9b361e979a95813a Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Jul 2026 00:10:27 +0800 Subject: [PATCH 14/34] [fix](file scanner) Address parquet validation and profile review --- be/src/format_v2/jni/jdbc_reader.cpp | 19 ++-- be/src/format_v2/jni/jni_table_reader.cpp | 37 ++++++-- be/src/format_v2/jni/jni_table_reader.h | 1 + .../jni/trino_connector_jni_reader.cpp | 11 ++- be/src/format_v2/parquet/parquet_profile.cpp | 16 ++-- .../reader/native/column_chunk_reader.cpp | 22 +++++ .../parquet/reader/native/column_reader.cpp | 9 ++ .../parquet/reader/native/level_decoder.cpp | 12 ++- .../parquet/reader/native_column_reader.cpp | 19 +++- be/src/format_v2/table/hive_reader.cpp | 15 ++- be/src/format_v2/table/hudi_reader.cpp | 25 +++-- ...eberg_position_delete_sys_table_reader.cpp | 15 ++- ...iceberg_position_delete_sys_table_reader.h | 1 + be/src/format_v2/table/iceberg_reader.cpp | 41 +++++---- be/src/format_v2/table/paimon_reader.cpp | 22 ++++- .../format_v2/table/remote_doris_reader.cpp | 7 +- be/src/format_v2/table_reader.cpp | 29 +++--- be/src/format_v2/table_reader.h | 4 + .../file_reader/file_meta_cache_test.cpp | 2 +- .../format_v2/jni/jni_table_reader_test.cpp | 43 ++++++++- .../format_v2/parquet/native_decoder_test.cpp | 91 +++++++++++++++++++ .../format_v2/parquet/parquet_reader_test.cpp | 87 ++++++++++++++++++ ..._position_delete_sys_table_reader_test.cpp | 63 +++++++++++++ .../format_v2/table/iceberg_reader_test.cpp | 29 +++++- 24 files changed, 540 insertions(+), 80 deletions(-) create mode 100644 be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp diff --git a/be/src/format_v2/jni/jdbc_reader.cpp b/be/src/format_v2/jni/jdbc_reader.cpp index e0391f3a13a8f0..7d28134db4d7e3 100644 --- a/be/src/format_v2/jni/jdbc_reader.cpp +++ b/be/src/format_v2/jni/jdbc_reader.cpp @@ -39,12 +39,19 @@ std::string JdbcJniReader::connector_class() const { } Status JdbcJniReader::prepare_split(const format::SplitReadOptions& options) { - _jdbc_params.clear(); - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.table_format_type == "jdbc") { - _jdbc_params = std::map( - options.current_range.table_format_params.jdbc_params.begin(), - options.current_range.table_format_params.jdbc_params.end()); + { + // End these scopes before JniTableReader enters the same counters; nested use would count + // this JDBC parameter preparation twice instead of extending the common lifecycle total. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + _jdbc_params.clear(); + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.table_format_type == "jdbc") { + _jdbc_params = std::map( + options.current_range.table_format_params.jdbc_params.begin(), + options.current_range.table_format_params.jdbc_params.end()); + } } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/jni/jni_table_reader.cpp b/be/src/format_v2/jni/jni_table_reader.cpp index dd858e4a20de8e..7658a52fc86e0f 100644 --- a/be/src/format_v2/jni/jni_table_reader.cpp +++ b/be/src/format_v2/jni/jni_table_reader.cpp @@ -32,19 +32,31 @@ namespace doris::format { Status JniTableReader::init(TableReadOptions&& options) { RETURN_IF_ERROR(TableReader::init(std::move(options))); - _init_profile(); + { + // Base and derived scopes must not overlap on the same counter: RuntimeProfile timers add + // deltas, so nested use would double-count instead of extending lifecycle coverage. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + _init_profile(); + } SCOPED_TIMER(_connector_total_time); return Status::OK(); } Status JniTableReader::prepare_split(const SplitReadOptions& options) { SCOPED_TIMER(_connector_total_time); - // EOF belongs to the previous split. Keep it set after closing that split so repeated reads - // are idempotent, and clear it only when a new split is explicitly prepared. - _eof = false; - _current_range = options.current_range; - RETURN_IF_ERROR(validate_scan_range(options.current_range)); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + // EOF belongs to the previous split. Keep it set after closing that split so repeated reads + // are idempotent, and clear it only when a new split is explicitly prepared. + _eof = false; + _current_range = options.current_range; + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -115,7 +127,11 @@ Status JniTableReader::get_block(Block* output_block, bool* eos) { } Status JniTableReader::abort_split() { - RETURN_IF_ERROR(_close_jni_scanner()); + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + RETURN_IF_ERROR(_close_jni_scanner()); + } return TableReader::abort_split(); } @@ -352,7 +368,12 @@ Status JniTableReader::close() { if (_closed) { return Status::OK(); } - auto close_status = _close_jni_scanner(); + Status close_status; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); + close_status = _close_jni_scanner(); + } auto table_status = TableReader::close(); if (close_status.ok() && !table_status.ok()) { close_status = std::move(table_status); diff --git a/be/src/format_v2/jni/jni_table_reader.h b/be/src/format_v2/jni/jni_table_reader.h index 4e3f4934f26334..5e48270515f996 100644 --- a/be/src/format_v2/jni/jni_table_reader.h +++ b/be/src/format_v2/jni/jni_table_reader.h @@ -84,6 +84,7 @@ class JniTableReader : public TableReader { virtual Status _open_jni_scanner(); bool _reserve_split_profile_publication(); const std::vector& jni_columns() const { return _jni_columns; } + RuntimeProfile::Counter* connector_total_timer() const { return _connector_total_time; } TFileRangeDesc _current_range; private: diff --git a/be/src/format_v2/jni/trino_connector_jni_reader.cpp b/be/src/format_v2/jni/trino_connector_jni_reader.cpp index 11c9945c5dea16..4f0a3f55c3891e 100644 --- a/be/src/format_v2/jni/trino_connector_jni_reader.cpp +++ b/be/src/format_v2/jni/trino_connector_jni_reader.cpp @@ -84,8 +84,15 @@ Status TrinoConnectorJniReader::validate_scan_range(const TFileRangeDesc& range) } Status TrinoConnectorJniReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(validate_scan_range(options.current_range)); - RETURN_IF_ERROR(_set_spi_plugins_dir()); + { + // Plugin discovery can dominate a cold split. Use non-overlapping common scopes because + // the JNI base method subsequently enters the same RuntimeProfile counters. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + SCOPED_TIMER(connector_total_timer()); + RETURN_IF_ERROR(validate_scan_range(options.current_range)); + RETURN_IF_ERROR(_set_spi_plugins_dir()); + } return format::JniTableReader::prepare_split(options); } diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index a5f0cd51050f7c..3c5de0a0a494b9 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -32,12 +32,12 @@ void ParquetProfile::init(RuntimeProfile* profile) { total_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, parquet_profile, file_scan_profile::FILE_READER, 1); - // These counters are format-independent and can be reused when one scanner switches between - // Parquet and ORC splits; keep their single flat counter identity under FileReader. + // Row-group counters are part of the long-standing ParquetReader profile contract. Keep them + // below the format node so profile parsers and operators can attribute pruning to Parquet. filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsFiltered", TUnit::UNIT, - file_scan_profile::FILE_READER, 1); + parquet_profile, 1); filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_dictionary = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "RowGroupsFilteredByDictionary", TUnit::UNIT, parquet_profile, 1); filtered_row_groups_by_bloom_filter = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -45,13 +45,13 @@ void ParquetProfile::init(RuntimeProfile* profile) { filtered_row_groups_by_page_index = ADD_CHILD_COUNTER_WITH_LEVEL( profile, "RowGroupsFilteredByPageIndex", TUnit::UNIT, parquet_profile, 1); to_read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsReadNum", TUnit::UNIT, - file_scan_profile::FILE_READER, 1); + parquet_profile, 1); total_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowGroupsTotalNum", TUnit::UNIT, parquet_profile, 1); selected_row_ranges = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRowRanges", TUnit::UNIT, parquet_profile, 1); filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByGroup", TUnit::UNIT, - file_scan_profile::FILE_READER, 1); + parquet_profile, 1); filtered_page_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByPage", TUnit::UNIT, parquet_profile, 1); pages_skipped_by_data_page_filter = ADD_CHILD_COUNTER_WITH_LEVEL( @@ -98,8 +98,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); nested_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); - lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "FilteredRowsByLazyRead", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", + TUnit::UNIT, parquet_profile, 1); filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, file_scan_profile::FILE_READER, 1); raw_rows_read = diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 92f00e63cbeba0..7f6dd9254757f4 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -421,6 +421,18 @@ Status ColumnChunkReader::parse_page_header() { return Status::Corruption("Parquet data page value count {} exceeds column total {}", page_num_values, _metadata.num_values); } + if constexpr (!IN_COLLECTION) { + const size_t page_start_row = _page_reader->start_row(); + const size_t page_end_row = _page_reader->end_row(); + if (UNLIKELY(page_end_row < page_start_row || + static_cast(page_num_values) != page_end_row - page_start_row)) { + // Flat columns have exactly one physical value slot per logical row. Rejecting a + // divergent header/OffsetIndex span prevents every later page from shifting rows. + return Status::Corruption( + "Parquet flat data page has {} values for logical row range [{}, {})", + page_num_values, page_start_row, page_end_row); + } + } _remaining_rep_nums = page_num_values; _remaining_def_nums = page_num_values; _remaining_num_values = page_num_values; @@ -1058,6 +1070,16 @@ Status ColumnChunkReader::load_page_nested_rows( } _current_row += *result_rows; + if ((_page_reader->is_header_v2() || OFFSET_INDEX) && + UNLIKELY(_current_row != _page_reader->end_row())) { + // V2 and OffsetIndex advertise an exact logical row span. A page that exhausts its + // repetition levels without that many row starts would otherwise make the caller retry + // the same row forever. + return Status::Corruption( + "Parquet nested data page ended at row {}, expected page end row {}", _current_row, + _page_reader->end_row()); + } + auto need_check_cross_page = [&]() -> bool { return !OFFSET_INDEX && IN_COLLECTION && _remaining_rep_nums == 0 && !_page_reader->is_header_v2() && has_next_page(); diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 579f0f213f3a63..fd783fd32a0b34 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -829,6 +829,11 @@ Status ScalarColumnReader::_read_nested_column( size_t before_rep_level_sz = _rep_levels.size(); RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, &load_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && load_rows == 0 && !cross_page)) { + // A bounded V2/indexed page must advance at least one logical row; zero progress would + // leave both range cursors unchanged and spin forever on corrupt repetition levels. + return Status::Corruption("Parquet nested reader made no row progress"); + } RETURN_IF_ERROR(read_and_fill_data(before_rep_level_sz, _filter_map_index)); _filter_map_index += load_rows; while (cross_page) { @@ -899,6 +904,10 @@ Status ScalarColumnReader::read_column_levels(Filte size_t level_start = _rep_levels.size(); RETURN_IF_ERROR(_chunk_reader->load_page_nested_rows(_rep_levels, right_row - left_row, &loaded_rows, &cross_page)); + if (UNLIKELY(right_row > left_row && loaded_rows == 0 && !cross_page)) { + // Keep the levels-only path under the same forward-progress invariant as value reads. + return Status::Corruption("Parquet nested level reader made no row progress"); + } RETURN_IF_ERROR(consume_level_segment(level_start, _filter_map_index)); _filter_map_index += loaded_rows; while (cross_page) { diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp index e563d0c03ff0a8..52113c951b82c4 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "common/cast_set.h" #include "format/parquet/parquet_common.h" @@ -57,12 +58,17 @@ Status LevelDecoder::init(Slice* slice, tparquet::Encoding::type encoding, level break; } case tparquet::Encoding::BIT_PACKED: { - uint32_t num_bits = num_levels * _bit_width; - uint32_t num_bytes = BitUtil::RoundUpNumBytes(num_bits); + // Header counts are uint32_t and can overflow before the byte bound check. Widen first so + // a forged level count cannot wrap to a small slice and desynchronize following values. + const uint64_t num_bits = static_cast(num_levels) * _bit_width; + const size_t num_bytes = static_cast((num_bits + 7) / 8); if (num_bytes > slice->size) { return Status::Corruption("Wrong parquet level format"); } - _bit_packed_decoder = BitReader((uint8_t*)slice->data, num_bytes); + if (num_bytes > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet BIT_PACKED level stream is too large"); + } + _bit_packed_decoder = BitReader((uint8_t*)slice->data, cast_set(num_bytes)); slice->data += num_bytes; slice->size -= num_bytes; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index e3ed1895136c6f..55127ae8d47f6b 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -101,6 +101,14 @@ const FieldSchema* find_child_field(const FieldSchema& parent, const ParquetColu return field_it == parent.children.end() ? nullptr : &*field_it; } +void collect_physical_subtree_ids(const FieldSchema& field, std::set* ids) { + DORIS_CHECK(ids != nullptr); + ids->insert(field.get_column_id()); + for (const auto& child : field.children) { + collect_physical_subtree_ids(child, ids); + } +} + void collect_projected_ids(const ParquetColumnSchema& schema, const format::LocalColumnIndex* projection, const FieldSchema& native_field, std::set* ids) { @@ -115,8 +123,15 @@ void collect_projected_ids(const ParquetColumnSchema& schema, DORIS_CHECK(schema_it != schema.children.end()); const FieldSchema* child_field = find_child_field(native_field, **schema_it); DORIS_CHECK(child_field != nullptr); - ids->insert(child_field->get_column_id()); - collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + if (format::is_full_projection(&child_projection)) { + // A full child path is a request for its complete physical subtree. Keeping only the + // child group id makes its grandchildren SkipReadingReaders and silently defaults + // STRUCT fields (or breaks ARRAY/MAP shape invariants). + collect_physical_subtree_ids(*child_field, ids); + } else { + ids->insert(child_field->get_column_id()); + collect_projected_ids(**schema_it, &child_projection, *child_field, ids); + } } if (schema.kind == ParquetColumnSchemaKind::MAP) { DORIS_CHECK(!native_field.children.empty()); diff --git a/be/src/format_v2/table/hive_reader.cpp b/be/src/format_v2/table/hive_reader.cpp index 74a95fd43965cf..619e9959a1bfa4 100644 --- a/be/src/format_v2/table/hive_reader.cpp +++ b/be/src/format_v2/table/hive_reader.cpp @@ -102,11 +102,16 @@ void add_name_mapping(std::vector* name_mapping, const std::string& } // namespace Status HiveReader::prepare_split(const format::SplitReadOptions& options) { - if (options.current_split_format != _format) { - return Status::InternalError( - "Hive scan expects all splits to use the same file format, " - "initialized_format={}, current_split_format={}", - static_cast(_format), static_cast(options.current_split_format)); + { + // Keep derived validation visible without overlapping the base scopes on the same timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + if (options.current_split_format != _format) { + return Status::InternalError( + "Hive scan expects all splits to use the same file format, " + "initialized_format={}, current_split_format={}", + static_cast(_format), static_cast(options.current_split_format)); + } } return format::TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index 6c8917d867074c..f66be726fdcfee 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -28,13 +28,20 @@ namespace doris::format::hudi { Status HudiReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.hudi_params && - options.current_range.table_format_params.hudi_params.__isset.schema_id) { - _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.hudi_params && + options.current_range.table_format_params.hudi_params.__isset.schema_id) { + _split_schema_id = options.current_range.table_format_params.hudi_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -66,7 +73,13 @@ Status HudiHybridReader::init(format::TableReadOptions&& options) { } Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(_ensure_current_split_reader(options)); + { + // Child readers use these same counters, so time only dispatch/creation here and end the + // outer scopes before invoking the child to preserve single-counted totals. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(_ensure_current_split_reader(options)); + } DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); } diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp index 09de209f3bc66e..fc1d797173b0ab 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp @@ -161,6 +161,10 @@ Status IcebergPositionDeleteSysTableV2Reader::prepare_split( const format::SplitReadOptions& options) { RETURN_IF_ERROR(close()); RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + // The inner delete-file reader has distinct counters, so the outer preparation can safely + // contain its cache miss/open work without re-entering the same RuntimeProfile timer. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); _current_range = options.current_range; _has_split = true; return _init_split(); @@ -308,6 +312,15 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { std::vector projected_columns; RETURN_IF_ERROR(_build_delete_file_projected_columns(&projected_columns)); + static constexpr const char* kPositionReaderProfile = "IcebergPositionDeleteFileReader"; + if (_position_reader_profile == nullptr) { + _position_reader_profile = _scanner_profile->get_child(kPositionReaderProfile); + if (_position_reader_profile == nullptr) { + // The outer system-table reader calls the inner reader synchronously. Giving both the + // same profile would nest identical counter pointers and double-count every timer. + _position_reader_profile = _scanner_profile->create_child(kPositionReaderProfile); + } + } _position_reader = std::make_unique(); RETURN_IF_ERROR(_position_reader->init({ .projected_columns = std::move(projected_columns), @@ -316,7 +329,7 @@ Status IcebergPositionDeleteSysTableV2Reader::_init_position_delete_reader() { .scan_params = _scan_params, .io_ctx = _io_ctx, .runtime_state = _runtime_state, - .scanner_profile = _scanner_profile, + .scanner_profile = _position_reader_profile, .file_slot_descs = nullptr, .push_down_agg_type = TPushAggOp::type::NONE, .condition_cache_digest = 0, diff --git a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h index 9c802e726053cb..09756daaabe4da 100644 --- a/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h +++ b/be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h @@ -76,6 +76,7 @@ class IcebergPositionDeleteSysTableV2Reader final : public format::TableReader { const TIcebergDeleteFileDesc* _delete_file_desc = nullptr; DeleteFileKind _delete_file_kind = DeleteFileKind::POSITION_DELETE; std::unique_ptr _position_reader; + RuntimeProfile* _position_reader_profile = nullptr; std::vector _read_columns; ColumnPtr _partition_value; roaring::Roaring64Map _dv_positions; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index b6580a12dfcd0f..d770cc095404b3 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -44,6 +44,7 @@ #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/table_reader.h" #include "io/file_factory.h" +#include "util/debug_points.h" #include "util/url_coding.h" namespace doris::format::iceberg { @@ -225,25 +226,31 @@ Status IcebergTableReader::PositionDeleteRowsCollector::collect(const Block& blo } Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options) { - _row_lineage_columns = {}; - _iceberg_params.reset(); - _delete_predicates_initialized = false; - _position_delete_rows_storage.clear(); - _equality_delete_filters.clear(); - _split_cache = options.cache; - if (options.current_range.__isset.table_format_params && - options.current_range.table_format_params.__isset.iceberg_params) { - const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; - _iceberg_params = iceberg_params; - if (iceberg_params.__isset.first_row_id) { - _row_lineage_columns.first_row_id = iceberg_params.first_row_id; - } - if (iceberg_params.__isset.last_updated_sequence_number) { - _row_lineage_columns.last_updated_sequence_number = - iceberg_params.last_updated_sequence_number; + { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _row_lineage_columns = {}; + _iceberg_params.reset(); + _delete_predicates_initialized = false; + _position_delete_rows_storage.clear(); + _equality_delete_filters.clear(); + _split_cache = options.cache; + if (options.current_range.__isset.table_format_params && + options.current_range.table_format_params.__isset.iceberg_params) { + const auto& iceberg_params = options.current_range.table_format_params.iceberg_params; + _iceberg_params = iceberg_params; + if (iceberg_params.__isset.first_row_id) { + _row_lineage_columns.first_row_id = iceberg_params.first_row_id; + } + if (iceberg_params.__isset.last_updated_sequence_number) { + _row_lineage_columns.last_updated_sequence_number = + iceberg_params.last_updated_sequence_number; + } } } RETURN_IF_ERROR(TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -255,6 +262,8 @@ Status IcebergTableReader::prepare_split(const format::SplitReadOptions& options if (_is_table_level_count_active()) { return Status::OK(); } + DBUG_EXECUTE_IF("IcebergTableReader.prepare_split.before_delete_file_scan", + DBUG_RUN_CALLBACK()); RETURN_IF_ERROR(_init_delete_predicates(options.current_range.table_format_params)); return Status::OK(); } diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 0b3e410b87c0eb..869b2b1294a675 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -33,12 +33,19 @@ namespace doris::format::paimon { Status PaimonReader::prepare_split(const format::SplitReadOptions& options) { - _split_schema_id = -1; - const auto& paimon_params = options.current_range.table_format_params.paimon_params; - if (paimon_params.__isset.schema_id) { - _split_schema_id = paimon_params.schema_id; + { + // Derived schema selection is additive to, not nested around, the common base timers. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + _split_schema_id = -1; + const auto& paimon_params = options.current_range.table_format_params.paimon_params; + if (paimon_params.__isset.schema_id) { + _split_schema_id = paimon_params.schema_id; + } } RETURN_IF_ERROR(format::TableReader::prepare_split(options)); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); if (current_split_pruned()) { return Status::OK(); } @@ -93,7 +100,12 @@ Status PaimonHybridReader::init(format::TableReadOptions&& options) { } Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { - RETURN_IF_ERROR(_ensure_current_split_reader(options)); + { + // End the outer dispatch scopes before the selected child enters the shared counters. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(_ensure_current_split_reader(options)); + } DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); } diff --git a/be/src/format_v2/table/remote_doris_reader.cpp b/be/src/format_v2/table/remote_doris_reader.cpp index 6c38147f1eccdd..db278d9b0cedda 100644 --- a/be/src/format_v2/table/remote_doris_reader.cpp +++ b/be/src/format_v2/table/remote_doris_reader.cpp @@ -385,7 +385,12 @@ Status RemoteDorisReader::init(TableReadOptions&& options) { } Status RemoteDorisReader::prepare_split(const SplitReadOptions& options) { - RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + { + // Keep protocol validation visible while avoiding overlap with TableReader's own scopes. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + RETURN_IF_ERROR(validate_remote_doris_range(options.current_range)); + } return TableReader::prepare_split(options); } diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index 77a808e298bc69..f7865aad63bc8d 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -521,21 +521,7 @@ std::optional TableReader::_find_current_table_column_by_field } Status TableReader::init(TableReadOptions&& options) { - _scan_params = options.scan_params; - _format = options.format; - _io_ctx = options.io_ctx; - _runtime_state = options.runtime_state; _scanner_profile = options.scanner_profile; - _file_slot_descs = options.file_slot_descs; - _push_down_agg_type = options.push_down_agg_type; - _push_down_count_columns = options.push_down_count_columns; - _initial_condition_cache_digest = options.condition_cache_digest; - _condition_cache_digest = _initial_condition_cache_digest; - _projected_columns = std::move(options.projected_columns); - _system_properties = create_system_properties(_scan_params); - _mapper_options.mode = TableColumnMappingMode::BY_NAME; - _conjuncts = std::move(options.conjuncts); - if (_scanner_profile != nullptr) { const auto hierarchy = file_scan_profile::ensure_hierarchy(_scanner_profile); static const char* table_profile = file_scan_profile::TABLE_READER; @@ -600,8 +586,23 @@ Status TableReader::init(TableReadOptions&& options) { _profile.file_reader_close_timer = ADD_CHILD_TIMER_WITH_LEVEL( _scanner_profile, "FileReaderCloseTime", file_reader_profile, 1); } + // Establish lifecycle timers before consuming options or constructing filesystem properties; + // placing these scopes at the tail records only scope teardown and hides expensive init work. SCOPED_TIMER(_profile.total_timer); SCOPED_TIMER(_profile.init_timer); + _scan_params = options.scan_params; + _format = options.format; + _io_ctx = options.io_ctx; + _runtime_state = options.runtime_state; + _file_slot_descs = options.file_slot_descs; + _push_down_agg_type = options.push_down_agg_type; + _push_down_count_columns = options.push_down_count_columns; + _initial_condition_cache_digest = options.condition_cache_digest; + _condition_cache_digest = _initial_condition_cache_digest; + _projected_columns = std::move(options.projected_columns); + _system_properties = create_system_properties(_scan_params); + _mapper_options.mode = TableColumnMappingMode::BY_NAME; + _conjuncts = std::move(options.conjuncts); return Status::OK(); } diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index dadd528e243e70..e87d8c48946deb 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -230,6 +230,10 @@ class TableReader { // stale external-table file listing that returns NOT_FOUND. The next prepare_split() must start // with no concrete reader or split-local state left from the failed split. virtual Status abort_split() { + // Ignored open failures still spend time closing partially initialized readers. Include + // that recovery path in the common lifecycle profile so NOT_FOUND cannot become invisible. + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.close_timer); if (_data_reader.reader != nullptr) { RETURN_IF_ERROR(close_current_reader()); } else { diff --git a/be/test/format/file_reader/file_meta_cache_test.cpp b/be/test/format/file_reader/file_meta_cache_test.cpp index fc5385dd56972a..4f8f803bd3ef28 100644 --- a/be/test/format/file_reader/file_meta_cache_test.cpp +++ b/be/test/format/file_reader/file_meta_cache_test.cpp @@ -155,4 +155,4 @@ TEST(FileMetaCacheTest, InsertAndLookupWithIntValue) { EXPECT_EQ(*cached_val2, 12345); } -} // namespace doris +} // namespace doris \ No newline at end of file diff --git a/be/test/format_v2/jni/jni_table_reader_test.cpp b/be/test/format_v2/jni/jni_table_reader_test.cpp index e3e5e69b825394..95eda4a557e790 100644 --- a/be/test/format_v2/jni/jni_table_reader_test.cpp +++ b/be/test/format_v2/jni/jni_table_reader_test.cpp @@ -19,10 +19,12 @@ #include +#include #include #include #include #include +#include #include #include "io/io_common.h" @@ -40,9 +42,15 @@ class FakeJniTableReader final : public JniTableReader { std::vector propagated_batch_sizes; std::vector close_results; bool next_eof = false; + std::chrono::milliseconds init_delay {0}; + std::chrono::milliseconds open_delay {0}; + std::chrono::milliseconds close_delay {0}; protected: - std::string connector_class() const override { return "test/FakeJniScanner"; } + std::string connector_class() const override { + std::this_thread::sleep_for(init_delay); + return "test/FakeJniScanner"; + } Status build_scanner_params(std::map* params) const override { params->clear(); @@ -57,6 +65,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _close_jni_scanner() override { + std::this_thread::sleep_for(close_delay); if (!TEST_scanner_opened()) { return Status::OK(); } @@ -80,6 +89,7 @@ class FakeJniTableReader final : public JniTableReader { } Status _open_jni_scanner() override { + std::this_thread::sleep_for(open_delay); open_batch_sizes.push_back(TEST_batch_size()); TEST_set_split_state(true, false); return Status::OK(); @@ -191,5 +201,36 @@ TEST(JniTableReaderTest, AdaptiveProbeSetBeforePrepareControlsFirstJniOpen) { EXPECT_TRUE(reader.TEST_scanner_opened()); } +TEST(JniTableReaderTest, CommonLifecycleTimersContainJniLifecycleWork) { + constexpr auto delay = std::chrono::milliseconds(8); + RuntimeProfile profile("JniLifecycleContainment"); + FakeJniTableReader reader; + reader.init_delay = delay; + ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok()); + ASSERT_GE(profile.get_counter("InitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.open_delay = delay; + ASSERT_TRUE(reader.prepare_split({ + .partition_values = {}, + .conjuncts = std::nullopt, + .partition_prune_conjuncts = {}, + .all_runtime_filters_applied = true, + .condition_cache_digest = std::nullopt, + .cache = nullptr, + .current_range = {}, + .current_split_format = FileFormat::JNI, + .global_rowid_context = std::nullopt, + }) + .ok()); + ASSERT_GE(profile.get_counter("PrepareSplitTime")->value(), + std::chrono::duration_cast(delay).count()); + + reader.close_delay = delay; + ASSERT_TRUE(reader.close().ok()); + EXPECT_GE(profile.get_counter("CloseTime")->value(), + std::chrono::duration_cast(delay).count()); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index c823abb2812702..08d7e61ea1caa8 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -747,6 +747,17 @@ TEST(ParquetV2NativeDecoderTest, BitPackedLevelCursorOperationsPreservePosition) EXPECT_EQ(rle_decoder.get_levels(&truncated_value, 1), 0); } +TEST(ParquetV2NativeDecoderTest, BitPackedLevelByteCountDoesNotWrapAtLargeCounts) { + char placeholder[8] = {}; + constexpr uint32_t num_levels = 1'500'000'000; + constexpr size_t expected_bytes = 562'500'000; + Slice levels(placeholder, expected_bytes); + LevelDecoder decoder; + + ASSERT_TRUE(decoder.init(&levels, tparquet::Encoding::BIT_PACKED, 4, num_levels).ok()); + EXPECT_EQ(levels.size, 0); +} + TEST(ParquetV2NativeDecoderTest, RejectsLevelsAboveSchemaMaximumOnEveryDecodePath) { auto make_decoder = [](tparquet::Encoding::type encoding) { static char encoded[] = {0x03}; // width=2 value=3, while the schema maximum is 2. @@ -1228,6 +1239,86 @@ TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption EXPECT_TRUE(parse_header(impossible_counts).is()); } +TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) { + auto init_chunk = [](tparquet::PageHeader header, bool with_offset_index) { + std::vector payload(static_cast(header.compressed_page_size), 0); + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(2); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + FieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 0; + field.definition_level = 0; + ParquetPageReadContext context(false, ""); + tparquet::OffsetIndex offset_index; + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(bytes.size()); + location.__set_first_row_index(0); + offset_index.__set_page_locations({location}); + if (with_offset_index) { + ColumnChunkReader reader_with_index(&reader, &chunk, &field, &offset_index, + 1, nullptr, context); + return reader_with_index.init(); + } + ColumnChunkReader sequential_reader(&reader, &chunk, &field, nullptr, 1, + nullptr, context); + return sequential_reader.init(); + }; + + tparquet::PageHeader v2; + v2.type = tparquet::PageType::DATA_PAGE_V2; + v2.__set_compressed_page_size(8); + v2.__set_uncompressed_page_size(8); + v2.__isset.data_page_header_v2 = true; + v2.data_page_header_v2.__set_num_values(2); + v2.data_page_header_v2.__set_num_rows(1); + v2.data_page_header_v2.__set_num_nulls(0); + v2.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + v2.data_page_header_v2.__set_repetition_levels_byte_length(0); + v2.data_page_header_v2.__set_definition_levels_byte_length(0); + v2.data_page_header_v2.__set_is_compressed(false); + auto status = init_chunk(v2, false); + EXPECT_TRUE(status.is()) << status; + status = init_chunk(v2, true); + EXPECT_TRUE(status.is()) << status; + + tparquet::PageHeader v1; + v1.type = tparquet::PageType::DATA_PAGE; + v1.__set_compressed_page_size(8); + v1.__set_uncompressed_page_size(8); + v1.__isset.data_page_header = true; + v1.data_page_header.__set_num_values(2); + v1.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + v1.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + v1.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + status = init_chunk(v1, true); + EXPECT_TRUE(status.is()) << status; +} + +TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(12); + header.__set_uncompressed_page_size(12); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(2); + header.data_page_header_v2.__set_num_rows(2); + header.data_page_header_v2.__set_num_nulls(0); + header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header_v2.__set_repetition_levels_byte_length(4); + header.data_page_header_v2.__set_definition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + // Two values [0, 1] contain only one repetition-level zero, so they describe one row. + EXPECT_TRUE(load_malformed_nested_page(header, {2, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0}, 2) + .is()); +} + TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { tparquet::PageHeader header; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index d68e8522e0ad15..77df0b43735725 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -62,6 +62,7 @@ #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_scan.h" #include "format_v2/parquet/reader/column_reader.h" +#include "format_v2/schema_projection.h" #include "format_v2/table_reader.h" #include "gen_cpp/Types_types.h" #include "io/io_common.h" @@ -775,6 +776,34 @@ void write_nullable_struct_with_list_parquet_file(const std::string& file_path) ROW_COUNT, builder.build())); } +void write_nested_complex_under_struct_parquet_file(const std::string& file_path) { + auto nested_struct = build_nullable_string_struct_array(); + auto nested_array = build_nullable_string_list_array(); + auto nested_map = build_nullable_int_string_map_array(); + auto marker = build_int32_array({10, 20, 30, 40, 50}); + auto struct_field = arrow::field("nested_struct", nested_struct->type(), true); + auto array_field = arrow::field("nested_array", nested_array->type(), true); + auto map_field = arrow::field("nested_map", nested_map->type(), true); + auto marker_field = arrow::field("marker", arrow::int32(), false); + auto outer_result = + arrow::StructArray::Make({nested_struct, nested_array, nested_map, marker}, + {struct_field, array_field, map_field, marker_field}); + ASSERT_TRUE(outer_result.ok()) << outer_result.status(); + auto outer = *outer_result; + auto table = arrow::Table::Make(arrow::schema({arrow::field("outer", outer->type(), false)}), + {outer}); + + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.version(::parquet::ParquetVersion::PARQUET_2_6); + builder.data_page_version(::parquet::ParquetDataPageVersion::V2); + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, + ROW_COUNT, builder.build())); +} + void write_int96_timestamp_parquet_file(const std::string& file_path) { auto field = arrow::field("ts_tz", arrow::timestamp(arrow::TimeUnit::MICRO), true); auto array = @@ -1438,6 +1467,62 @@ TEST_F(NewParquetReaderTest, NativeComplexColumnsMaterializeDirectlyAcrossBatchC EXPECT_GT(profile.get_counter("NestedBatches")->value(), 0); } +TEST_F(NewParquetReaderTest, FullComplexChildUnderPartialParentReadsItsWholeSubtree) { + write_nested_complex_under_struct_parquet_file(_file_path); + for (size_t child_index = 0; child_index < 3; ++child_index) { + auto reader = create_reader(); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 1); + ASSERT_EQ(schema[0].children.size(), 4); + auto projection = format::LocalColumnIndex::partial_local(0); + projection.children.push_back( + format::LocalColumnIndex::local(schema[0].children[child_index].file_local_id())); + auto request = std::make_shared(); + request->non_predicate_columns = {projection}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + ASSERT_TRUE(reader->open(request).ok()); + + format::ColumnDefinition projected; + ASSERT_TRUE(format::project_column_definition(schema[0], projection, &projected).ok()); + Block block; + block.insert(ColumnWithTypeAndName(projected.type->create_column(), projected.type, + projected.name)); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_GT(rows, 0); + const auto& outer = nullable_nested_column(block, 0); + ASSERT_EQ(outer.tuple_size(), 1); + const auto& nullable_nested = assert_cast(outer.get_column(0)); + if (child_index == 0) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + ASSERT_EQ(nested.tuple_size(), 2); + const auto& payload = assert_cast(nested.get_column(0)); + EXPECT_EQ(assert_cast(payload.get_nested_column()) + .get_data_at(0) + .to_string(), + "small"); + const auto& ids = assert_cast(nested.get_column(1)); + EXPECT_EQ(ids.get_nested_column().get_int(0), 1); + } else if (child_index == 1) { + const auto& nested = + assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 2); + EXPECT_EQ(nested.get_data().size(), 4); + } else { + const auto& nested = assert_cast(nullable_nested.get_nested_column()); + EXPECT_EQ(nested.get_offsets()[0], 1); + EXPECT_EQ(nested.get_keys().size(), 3); + EXPECT_EQ(nested.get_values().size(), 3); + } + } +} + TEST_F(NewParquetReaderTest, NativeNestedMapUsesOuterKeyRepetitionShape) { const char* source_root = std::getenv("ROOT"); ASSERT_NE(source_root, nullptr); @@ -2115,6 +2200,8 @@ TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTo EXPECT_TRUE(children.at("FileReader").contains("ParquetReader")); ASSERT_TRUE(children.contains("ParquetReader")); EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowGroupsReadNum")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredRowsByGroup")); } TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { diff --git a/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp new file mode 100644 index 00000000000000..a219ba37f3dd8d --- /dev/null +++ b/be/test/format_v2/table/iceberg_position_delete_sys_table_reader_test.cpp @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/table/iceberg_position_delete_sys_table_reader.h" + +#include + +#include "runtime/runtime_profile.h" +#include "runtime/runtime_state.h" + +namespace doris::format::iceberg { +namespace { + +TFileRangeDesc range_with_delete_file(const TIcebergDeleteFileDesc& delete_file) { + TIcebergFileDesc iceberg_desc; + iceberg_desc.__set_delete_files({delete_file}); + TTableFormatFileDesc table_format_desc; + table_format_desc.__set_iceberg_params(std::move(iceberg_desc)); + TFileRangeDesc range; + range.__set_table_format_params(std::move(table_format_desc)); + return range; +} + +TEST(IcebergPositionDeleteSysTableV2ProfileTest, UsesDistinctProfileForNestedPositionReader) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile profile("position_delete_system_table_profile"); + TFileScanRangeParams params; + std::vector file_slot_descs; + TIcebergDeleteFileDesc delete_file; + delete_file.__set_content(1); + delete_file.__set_file_format(TFileFormatType::FORMAT_PARQUET); + delete_file.__set_path("/not-opened-during-prepare.parquet"); + + IcebergPositionDeleteSysTableV2Reader reader; + reader._runtime_state = &state; + reader._scanner_profile = &profile; + reader._scan_params = ¶ms; + reader._file_slot_descs = &file_slot_descs; + reader._current_range = range_with_delete_file(delete_file); + ASSERT_TRUE(reader._init_split().ok()); + + ASSERT_NE(reader._position_reader_profile, nullptr); + EXPECT_NE(reader._position_reader_profile, &profile); + EXPECT_EQ(profile.get_child("IcebergPositionDeleteFileReader"), + reader._position_reader_profile); +} + +} // namespace +} // namespace doris::format::iceberg diff --git a/be/test/format_v2/table/iceberg_reader_test.cpp b/be/test/format_v2/table/iceberg_reader_test.cpp index 71dc1ea1db7c69..7d8572101e342e 100644 --- a/be/test/format_v2/table/iceberg_reader_test.cpp +++ b/be/test/format_v2/table/iceberg_reader_test.cpp @@ -25,12 +25,15 @@ #include #include +#include #include #include #include +#include #include #include #include +#include #include #include #include @@ -732,6 +735,12 @@ class ScopedDebugPoint { DebugPoints::instance()->add(_name); } + ScopedDebugPoint(std::string name, std::function callback) + : _name(std::move(name)), _enable_debug_points(config::enable_debug_points) { + config::enable_debug_points = true; + DebugPoints::instance()->add_with_callback(_name, std::move(callback)); + } + ~ScopedDebugPoint() { DebugPoints::instance()->remove(_name); config::enable_debug_points = _enable_debug_points; @@ -2934,7 +2943,25 @@ TEST(IcebergV2ReaderTest, IcebergPositionDeleteFileIsReusedAcrossSplits) { first_split.cache = &cache; first_split.current_range.__set_table_format_params(make_iceberg_table_format_desc( first_file_path, {make_iceberg_position_delete_file(delete_file_path)})); - ASSERT_TRUE(reader.prepare_split(first_split).ok()); + const auto* total_timer = profile.get_counter("TableReader"); + const auto* prepare_timer = profile.get_counter("PrepareSplitTime"); + ASSERT_NE(total_timer, nullptr); + ASSERT_NE(prepare_timer, nullptr); + const int64_t total_before = total_timer->value(); + const int64_t prepare_before = prepare_timer->value(); + constexpr int64_t DELETE_FILE_DELAY_NS = 8'000'000; + { + ScopedDebugPoint delay_delete_file_scan( + "IcebergTableReader.prepare_split.before_delete_file_scan", + [] { std::this_thread::sleep_for(std::chrono::milliseconds(8)); }); + ASSERT_TRUE(reader.prepare_split(first_split).ok()); + } + const int64_t total_delta = total_timer->value() - total_before; + const int64_t prepare_delta = prepare_timer->value() - prepare_before; + // A cache-miss delete-file scan is derived prepare work; both common parent timers must + // contain it so the expensive miss cannot disappear between profile levels. + EXPECT_GE(prepare_delta, DELETE_FILE_DELAY_NS); + EXPECT_GE(total_delta, DELETE_FILE_DELAY_NS); EXPECT_EQ(read_iceberg_ids(&reader, projected_columns), std::vector({1, 3})); // The cached delete file contains entries for every referenced data file, so another split can From ed40395fbcc0df51566ce2dde5294bc5cbbd7914 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Jul 2026 19:54:09 +0800 Subject: [PATCH 15/34] [fix](be) make parquet v2 metadata path native --- .../format_v2/parquet/native_schema_desc.cpp | 729 ++++++++++++++++++ be/src/format_v2/parquet/native_schema_desc.h | 171 ++++ .../format_v2/parquet/native_schema_node.cpp | 127 +++ be/src/format_v2/parquet/native_schema_node.h | 93 +++ .../parquet/parquet_column_schema.cpp | 220 ++++++ .../format_v2/parquet/parquet_column_schema.h | 5 + .../parquet/parquet_file_context.cpp | 176 ++--- .../format_v2/parquet/parquet_file_context.h | 48 +- be/src/format_v2/parquet/parquet_profile.cpp | 10 +- be/src/format_v2/parquet/parquet_profile.h | 2 - be/src/format_v2/parquet/parquet_reader.cpp | 73 +- be/src/format_v2/parquet/parquet_scan.cpp | 217 +++++- be/src/format_v2/parquet/parquet_scan.h | 11 +- .../format_v2/parquet/parquet_statistics.cpp | 409 +++++++++- be/src/format_v2/parquet/parquet_statistics.h | 13 + .../parquet/reader/count_column_reader.cpp | 6 +- .../reader/native/column_chunk_reader.cpp | 166 +++- .../reader/native/column_chunk_reader.h | 12 +- .../parquet/reader/native/column_reader.cpp | 191 +++-- .../parquet/reader/native/column_reader.h | 42 +- .../native/fix_length_plain_decoder.cpp | 8 +- .../parquet/reader/native/level_reader.cpp | 12 +- .../parquet/reader/native/level_reader.h | 9 +- .../parquet/reader/native_column_reader.cpp | 20 +- .../parquet/reader/native_column_reader.h | 6 +- .../format_v2/parquet/native_decoder_test.cpp | 132 +++- .../parquet/parquet_page_cache_range_test.cpp | 13 + .../format_v2/parquet/parquet_reader_test.cpp | 7 + .../format_v2/parquet/parquet_schema_test.cpp | 94 +++ .../parquet/parquet_statistics_test.cpp | 34 + docs/file-scanner-v2-code-review-guide.md | 17 +- docs/file-scanner-v2-parquet-scan-design.md | 48 +- .../tvf/test_hdfs_parquet_group6.out | 16 +- .../nereids_rules_p0/pkfk/eliminate_inner.out | 16 +- 34 files changed, 2707 insertions(+), 446 deletions(-) create mode 100644 be/src/format_v2/parquet/native_schema_desc.cpp create mode 100644 be/src/format_v2/parquet/native_schema_desc.h create mode 100644 be/src/format_v2/parquet/native_schema_node.cpp create mode 100644 be/src/format_v2/parquet/native_schema_node.h diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp new file mode 100644 index 00000000000000..f0b93e4e030a20 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -0,0 +1,729 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_desc.h" + +#include + +#include +#include +#include + +#include "common/cast_set.h" +#include "common/exception.h" +#include "common/logging.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_factory.hpp" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_struct.h" +#include "core/data_type/define_primitive_type.h" +#include "util/slice.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +static bool is_group_node(const tparquet::SchemaElement& schema) { + return schema.num_children > 0; +} + +static bool is_list_node(const tparquet::SchemaElement& schema) { + return schema.__isset.converted_type && schema.converted_type == tparquet::ConvertedType::LIST; +} + +static bool is_map_node(const tparquet::SchemaElement& schema) { + return schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::MAP || + schema.converted_type == tparquet::ConvertedType::MAP_KEY_VALUE); +} + +static bool is_repeated_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REPEATED; +} + +static bool is_required_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::REQUIRED; +} + +static bool is_optional_node(const tparquet::SchemaElement& schema) { + return schema.__isset.repetition_type && + schema.repetition_type == tparquet::FieldRepetitionType::OPTIONAL; +} + +static int num_children_node(const tparquet::SchemaElement& schema) { + return schema.__isset.num_children ? schema.num_children : 0; +} + +/** + * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. + * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the + * empty or null values in ancestor node. + * + * For instance, considering an array of strings with 3 rows like the following: + * null, [], [a, b, c] + * We can store four elements in data column: null, a, b, c + * and the offsets column is: 1, 1, 4 + * and the null map is: 1, 0, 0 + * For the i-th row in array column: range from `offsets[i - 1]` until `offsets[i]` represents the elements in this row, + * so we can't store empty array/map values in doris data column. + * As a comparison, spark does not require `repeated_parent_def_level`, + * because the spark column stores empty array/map values , and use anther length column to indicate empty values. + * Please reference: https://github.com/apache/spark/blob/master/sql/core/src/main/java/org/apache/spark/sql/execution/datasources/parquet/ParquetColumnVector.java + * + * Furthermore, we can also avoid store null array/map values in doris data column. + * The same three rows as above, We can only store three elements in data column: a, b, c + * and the offsets column is: 0, 0, 3 + * and the null map is: 1, 0, 0 + * + * Inherit the repetition and definition level from parent node, if the parent node is repeated, + * we should set repeated_parent_def_level = definition_level, otherwise as repeated_parent_def_level. + * @param parent parent node + * @param repeated_parent_def_level the first ancestor node whose repetition_type equals REPEATED + */ +static void set_child_node_level(NativeFieldSchema* parent, int16_t repeated_parent_def_level) { + for (auto& child : parent->children) { + child.repetition_level = parent->repetition_level; + child.definition_level = parent->definition_level; + child.repeated_parent_def_level = repeated_parent_def_level; + } +} + +static bool is_struct_list_node(const tparquet::SchemaElement& schema) { + const std::string& name = schema.name; + static const Slice array_slice("array", 5); + static const Slice tuple_slice("_tuple", 6); + Slice slice(name); + return slice == array_slice || slice.ends_with(tuple_slice); +} + +std::string NativeFieldSchema::debug_string() const { + std::stringstream ss; + ss << "NativeFieldSchema(name=" << name << ", R=" << repetition_level + << ", D=" << definition_level; + if (children.size() > 0) { + ss << ", type=" << data_type->get_name() << ", children=["; + for (int i = 0; i < children.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << children[i].debug_string(); + } + ss << "]"; + } else { + ss << ", physical_type=" << physical_type; + ss << " , doris_type=" << data_type->get_name(); + } + ss << ")"; + return ss.str(); +} + +Status NativeFieldDescriptor::parse_from_thrift( + const std::vector& t_schemas) { + if (t_schemas.size() == 0 || !is_group_node(t_schemas[0])) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + const auto& root_schema = t_schemas[0]; + _fields.resize(root_schema.num_children); + _next_schema_pos = 1; + + for (int i = 0; i < root_schema.num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &_fields[i])); + if (_name_to_field.find(_fields[i].name) != _name_to_field.end()) { + return Status::InvalidArgument("Duplicated field name: {}", _fields[i].name); + } + _name_to_field.emplace(_fields[i].name, &_fields[i]); + } + + if (_next_schema_pos != t_schemas.size()) { + return Status::InvalidArgument("Remaining {} unparsed schema elements", + t_schemas.size() - _next_schema_pos); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_node_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field) { + if (curr_pos >= t_schemas.size()) { + return Status::InvalidArgument("Out-of-bounds index of schema elements"); + } + auto& t_schema = t_schemas[curr_pos]; + if (is_group_node(t_schema)) { + // nested structure or nullable list + return parse_group_field(t_schemas, curr_pos, node_field); + } + if (is_repeated_node(t_schema)) { + // repeated (LIST) + // produce required list + node_field->repetition_level++; + node_field->definition_level++; + node_field->children.resize(1); + set_child_node_level(node_field, node_field->definition_level); + auto child = &node_field->children[0]; + parse_physical_field(t_schema, false, child); + + node_field->name = t_schema.name; + node_field->lower_case_name = to_lower(t_schema.name); + node_field->data_type = std::make_shared(make_nullable(child->data_type)); + _next_schema_pos = curr_pos + 1; + node_field->field_id = t_schema.__isset.field_id ? t_schema.field_id : -1; + } else { + bool is_optional = is_optional_node(t_schema); + if (is_optional) { + node_field->definition_level++; + } + parse_physical_field(t_schema, is_optional, node_field); + _next_schema_pos = curr_pos + 1; + } + return Status::OK(); +} + +void NativeFieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema, + bool is_nullable, + NativeFieldSchema* physical_field) { + physical_field->name = physical_schema.name; + physical_field->lower_case_name = to_lower(physical_field->name); + physical_field->parquet_schema = physical_schema; + physical_field->physical_type = physical_schema.type; + physical_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + _physical_fields.push_back(physical_field); + physical_field->physical_column_index = cast_set(_physical_fields.size() - 1); + auto type = get_doris_type(physical_schema, is_nullable); + physical_field->data_type = type.first; + physical_field->is_type_compatibility = type.second; + physical_field->field_id = physical_schema.__isset.field_id ? physical_schema.field_id : -1; +} + +std::pair NativeFieldDescriptor::get_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + try { + if (physical_schema.__isset.logicalType) { + ans = convert_to_doris_type(physical_schema.logicalType, nullable); + } else if (physical_schema.__isset.converted_type) { + ans = convert_to_doris_type(physical_schema, nullable); + } + } catch (...) { + // now the Not supported exception are ignored + // so those byte_array maybe be treated as varbinary(now) : string(before) + } + if (ans.first->get_primitive_type() == PrimitiveType::INVALID_TYPE) { + switch (physical_schema.type) { + case tparquet::Type::BOOLEAN: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BOOLEAN, nullable); + break; + case tparquet::Type::INT32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::Type::INT64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::Type::INT96: + if (_enable_mapping_timestamp_tz) { + // treat INT96 as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMESTAMPTZ, nullable, + 0, 6); + } else { + // in most cases, it's a nano timestamp + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, + 0, 6); + } + break; + case tparquet::Type::FLOAT: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + break; + case tparquet::Type::DOUBLE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DOUBLE, nullable); + break; + case tparquet::Type::BYTE_ARRAY: + if (_enable_mapping_varbinary) { + // if physical_schema not set logicalType and converted_type, + // we treat BYTE_ARRAY as VARBINARY by default, so that we can read all data directly. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + break; + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet logicalType{}", + physical_schema.type)); + break; + } + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + tparquet::LogicalType logicalType, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + if (logicalType.__isset.STRING) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } else if (logicalType.__isset.DECIMAL) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, + logicalType.DECIMAL.precision, + logicalType.DECIMAL.scale); + } else if (logicalType.__isset.DATE) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + } else if (logicalType.__isset.INTEGER) { + if (logicalType.INTEGER.isSigned) { + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } + } else { + is_type_compatibility = true; + if (logicalType.INTEGER.bitWidth <= 8) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + } else if (logicalType.INTEGER.bitWidth <= 32) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + } + } + } else if (logicalType.__isset.TIME) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); + } else if (logicalType.__isset.TIMESTAMP) { + if (_enable_mapping_timestamp_tz) { + if (logicalType.TIMESTAMP.isAdjustedToUTC) { + // treat TIMESTAMP with isAdjustedToUTC as TIMESTAMPTZ + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_TIMESTAMPTZ, nullable, 0, + logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + return ans; + } + } + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); + } else if (logicalType.__isset.JSON) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } else if (logicalType.__isset.UUID) { + if (_enable_mapping_varbinary) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, + -1, 16); + } else { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + } + } else if (logicalType.__isset.FLOAT16) { + ans.first = DataTypeFactory::instance().create_data_type(TYPE_FLOAT, nullable); + } else { + throw Exception(Status::InternalError("Not supported parquet logicalType")); + } + return ans; +} + +std::pair NativeFieldDescriptor::convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable) { + std::pair ans = {std::make_shared(), false}; + bool& is_type_compatibility = ans.second; + switch (physical_schema.converted_type) { + case tparquet::ConvertedType::type::UTF8: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + case tparquet::ConvertedType::type::DECIMAL: + ans.first = DataTypeFactory::instance().create_data_type( + TYPE_DECIMAL128I, nullable, physical_schema.precision, physical_schema.scale); + break; + case tparquet::ConvertedType::type::DATE: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); + break; + case tparquet::ConvertedType::type::TIME_MILLIS: + [[fallthrough]]; + case tparquet::ConvertedType::type::TIME_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MILLIS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 3); + break; + case tparquet::ConvertedType::type::TIMESTAMP_MICROS: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 6); + break; + case tparquet::ConvertedType::type::INT_8: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TINYINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_8: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_16: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_SMALLINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_16: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_32: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_INT, nullable); + break; + case tparquet::ConvertedType::type::UINT_32: + is_type_compatibility = true; + [[fallthrough]]; + case tparquet::ConvertedType::type::INT_64: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_BIGINT, nullable); + break; + case tparquet::ConvertedType::type::UINT_64: + is_type_compatibility = true; + ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); + break; + case tparquet::ConvertedType::type::JSON: + ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); + break; + default: + throw Exception(Status::InternalError("Not supported parquet ConvertedType: {}", + physical_schema.converted_type)); + } + return ans; +} + +Status NativeFieldDescriptor::parse_group_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field) { + auto& group_schema = t_schemas[curr_pos]; + if ((group_schema.__isset.logicalType && group_schema.logicalType.__isset.ENUM) || + (group_schema.__isset.converted_type && + group_schema.converted_type == tparquet::ConvertedType::ENUM)) { + // ENUM describes primitive bytes only. Rejecting it before recursive group parsing keeps + // the native metadata tree from silently accepting a schema the Parquet contract forbids. + return Status::InvalidArgument("Logical type Enum cannot be applied to group node"); + } + if (is_map_node(group_schema)) { + // the map definition: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + return parse_map_field(t_schemas, curr_pos, group_field); + } + if (is_list_node(group_schema)) { + // the list definition: + // optional group (LIST) { + // repeated group [bag | list] { // hive or spark + // optional [array_element | element]; // hive or spark + // } + // } + return parse_list_field(t_schemas, curr_pos, group_field); + } + + if (is_repeated_node(group_schema)) { + group_field->repetition_level++; + group_field->definition_level++; + group_field->children.resize(1); + set_child_node_level(group_field, group_field->definition_level); + auto struct_field = &group_field->children[0]; + // the list of struct: + // repeated group (LIST) { + // optional/required ; + // ... + // } + // produce a non-null list + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field)); + + group_field->name = group_schema.name; + group_field->lower_case_name = to_lower(group_field->name); + group_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + group_field->data_type = + std::make_shared(make_nullable(struct_field->data_type)); + group_field->field_id = group_schema.__isset.field_id ? group_schema.field_id : -1; + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, group_field)); + } + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_list_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field) { + // the list definition: + // spark and hive have three level schemas but with different schema name + // spark: - "list" - "element" + // hive: - "bag" - "array_element" + // parse three level schemas to two level primitive like: LIST, + // or nested structure like: LIST> + auto& first_level = t_schemas[curr_pos]; + if (first_level.num_children != 1) { + return Status::InvalidArgument("List element should have only one child"); + } + + if (curr_pos + 1 >= t_schemas.size()) { + return Status::InvalidArgument("List element should have the second level schema"); + } + + if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED) { + return Status::InvalidArgument("List element can't be a repeated schema"); + } + + // the repeated schema element + auto& second_level = t_schemas[curr_pos + 1]; + if (second_level.repetition_type != tparquet::FieldRepetitionType::REPEATED) { + return Status::InvalidArgument("The second level of list element should be repeated"); + } + + // This indicates if this list is nullable. + bool is_optional = is_optional_node(first_level); + if (is_optional) { + list_field->definition_level++; + } + list_field->repetition_level++; + list_field->definition_level++; + list_field->children.resize(1); + NativeFieldSchema* list_child = &list_field->children[0]; + + size_t num_children = num_children_node(second_level); + if (num_children > 0) { + if (num_children == 1 && !is_struct_list_node(second_level)) { + // optional field, and the third level element is the nested structure in list + // produce nested structure like: LIST, LIST, LIST> + // skip bag/list, it's a repeated element. + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_node_field(t_schemas, curr_pos + 2, list_child)); + } else { + // required field, produce the list of struct + set_child_node_level(list_field, list_field->definition_level); + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 0) { + // required two level list, for compatibility reason. + set_child_node_level(list_field, list_field->definition_level); + parse_physical_field(second_level, false, list_child); + _next_schema_pos = curr_pos + 2; + } + + list_field->name = first_level.name; + list_field->lower_case_name = to_lower(first_level.name); + list_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + list_field->data_type = + std::make_shared(make_nullable(list_field->children[0].data_type)); + if (is_optional) { + list_field->data_type = make_nullable(list_field->data_type); + } + list_field->field_id = first_level.__isset.field_id ? first_level.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* map_field) { + // the map definition in parquet: + // optional group (MAP) { + // repeated group map (MAP_KEY_VALUE) { + // required key; + // optional value; + // } + // } + // Map value can be optional, the map without values is a SET + if (curr_pos + 2 >= t_schemas.size()) { + return Status::InvalidArgument("Map element should have at least three levels"); + } + auto& map_schema = t_schemas[curr_pos]; + if (map_schema.num_children != 1) { + return Status::InvalidArgument( + "Map element should have only one child(name='map', type='MAP_KEY_VALUE')"); + } + if (is_repeated_node(map_schema)) { + return Status::InvalidArgument("Map element can't be a repeated schema"); + } + auto& map_key_value = t_schemas[curr_pos + 1]; + if (!is_group_node(map_key_value) || !is_repeated_node(map_key_value)) { + return Status::InvalidArgument( + "the second level in map must be a repeated group(key and value)"); + } + auto& map_key = t_schemas[curr_pos + 2]; + if (!is_required_node(map_key)) { + LOG(WARNING) << "Filed " << map_schema.name << " is map type, but with nullable key column"; + } + + if (map_key_value.num_children == 1) { + // The map with three levels is a SET + return parse_list_field(t_schemas, curr_pos, map_field); + } + if (map_key_value.num_children != 2) { + // A standard map should have four levels + return Status::InvalidArgument( + "the second level in map(MAP_KEY_VALUE) should have two children"); + } + // standard map + bool is_optional = is_optional_node(map_schema); + if (is_optional) { + map_field->definition_level++; + } + map_field->repetition_level++; + map_field->definition_level++; + + // Directly create key and value children instead of intermediate key_value node + map_field->children.resize(2); + // map is a repeated node, we should set the `repeated_parent_def_level` of its children as `definition_level` + set_child_node_level(map_field, map_field->definition_level); + + auto key_field = &map_field->children[0]; + auto value_field = &map_field->children[1]; + + // Parse key and value fields directly from the key_value group's children + _next_schema_pos = curr_pos + 2; // Skip key_value group, go directly to key + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, key_field)); + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, value_field)); + + map_field->name = map_schema.name; + map_field->lower_case_name = to_lower(map_field->name); + map_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + map_field->data_type = std::make_shared(make_nullable(key_field->data_type), + make_nullable(value_field->data_type)); + if (is_optional) { + map_field->data_type = make_nullable(map_field->data_type); + } + map_field->field_id = map_schema.__isset.field_id ? map_schema.field_id : -1; + + return Status::OK(); +} + +Status NativeFieldDescriptor::parse_struct_field( + const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* struct_field) { + // the nested column in parquet, parse group to struct. + auto& struct_schema = t_schemas[curr_pos]; + bool is_optional = is_optional_node(struct_schema); + if (is_optional) { + struct_field->definition_level++; + } + auto num_children = struct_schema.num_children; + struct_field->children.resize(num_children); + set_child_node_level(struct_field, struct_field->repeated_parent_def_level); + _next_schema_pos = curr_pos + 1; + for (int i = 0; i < num_children; ++i) { + RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i])); + } + struct_field->name = struct_schema.name; + struct_field->lower_case_name = to_lower(struct_field->name); + struct_field->column_id = NATIVE_UNASSIGNED_COLUMN_ID; // Initialize column_id + + struct_field->field_id = struct_schema.__isset.field_id ? struct_schema.field_id : -1; + DataTypes res_data_types; + std::vector names; + for (int i = 0; i < num_children; ++i) { + res_data_types.push_back(make_nullable(struct_field->children[i].data_type)); + names.push_back(struct_field->children[i].name); + } + struct_field->data_type = std::make_shared(res_data_types, names); + if (is_optional) { + struct_field->data_type = make_nullable(struct_field->data_type); + } + return Status::OK(); +} + +int NativeFieldDescriptor::get_column_index(const std::string& column) const { + for (int32_t i = 0; i < _fields.size(); i++) { + if (_fields[i].name == column) { + return i; + } + } + return -1; +} + +NativeFieldSchema* NativeFieldDescriptor::get_column(const std::string& name) const { + auto it = _name_to_field.find(name); + if (it != _name_to_field.end()) { + return it->second; + } + throw Exception(Status::InternalError("Name {} not found in NativeFieldDescriptor!", name)); + return nullptr; +} + +void NativeFieldDescriptor::get_column_names(std::unordered_set* names) const { + names->clear(); + for (const NativeFieldSchema& f : _fields) { + names->emplace(f.name); + } +} + +std::string NativeFieldDescriptor::debug_string() const { + std::stringstream ss; + ss << "fields=["; + for (int i = 0; i < _fields.size(); ++i) { + if (i != 0) { + ss << ", "; + } + ss << _fields[i].debug_string(); + } + ss << "]"; + return ss.str(); +} + +void NativeFieldDescriptor::assign_ids() { + uint64_t next_id = 1; + for (auto& field : _fields) { + field.assign_ids(next_id); + } +} + +const NativeFieldSchema* NativeFieldDescriptor::find_column_by_id(uint64_t column_id) const { + for (const auto& field : _fields) { + if (auto result = field.find_column_by_id(column_id)) { + return result; + } + } + return nullptr; +} + +void NativeFieldSchema::assign_ids(uint64_t& next_id) { + column_id = next_id++; + + for (auto& child : children) { + child.assign_ids(next_id); + } + + max_column_id = next_id - 1; +} + +const NativeFieldSchema* NativeFieldSchema::find_column_by_id(uint64_t target_id) const { + if (column_id == target_id) { + return this; + } + + for (const auto& child : children) { + if (auto result = child.find_column_by_id(target_id)) { + return result; + } + } + + return nullptr; +} + +uint64_t NativeFieldSchema::get_column_id() const { + return column_id; +} + +void NativeFieldSchema::set_column_id(uint64_t id) { + column_id = id; +} + +uint64_t NativeFieldSchema::get_max_column_id() const { + return max_column_id; +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h new file mode 100644 index 00000000000000..c74b3f9af02dc6 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "common/cast_set.h" +#include "common/status.h" +#include "core/data_type/data_type.h" +#include "core/data_type/data_type_nothing.h" +#include "util/slice.h" + +namespace doris::format::parquet { + +// Constant for unassigned column IDs +constexpr uint64_t NATIVE_UNASSIGNED_COLUMN_ID = UINT64_MAX; + +struct NativeFieldSchema { + std::string name; + std::string lower_case_name; // for hms column name case insensitive match + // the referenced parquet schema element + tparquet::SchemaElement parquet_schema; + + // Used to identify whether this field is a nested field. + DataTypePtr data_type; + + // Only valid when this field is a leaf node + tparquet::Type::type physical_type; + // The index order in NativeFieldDescriptor._physical_fields + int physical_column_index = -1; + int16_t definition_level = 0; + int16_t repetition_level = 0; + int16_t repeated_parent_def_level = 0; + std::vector children; + + //For UInt8 -> Int16,UInt16 -> Int32,UInt32 -> Int64,UInt64 -> Int128. + bool is_type_compatibility = false; + + NativeFieldSchema() + : data_type(std::make_shared()), + column_id(NATIVE_UNASSIGNED_COLUMN_ID) {} + ~NativeFieldSchema() = default; + NativeFieldSchema(const NativeFieldSchema& fieldSchema) = default; + std::string debug_string() const; + + int32_t field_id = -1; + uint64_t column_id = NATIVE_UNASSIGNED_COLUMN_ID; + uint64_t max_column_id = 0; // Maximum column ID for this field and its children + + // Column ID assignment and lookup methods + void assign_ids(uint64_t& next_id); + const NativeFieldSchema* find_column_by_id(uint64_t target_id) const; + uint64_t get_column_id() const; + void set_column_id(uint64_t id); + uint64_t get_max_column_id() const; +}; + +// V2 owns this schema tree and parser so footer/schema planning never invokes the V1 reader path. +class NativeFieldDescriptor { +private: + // Only the schema elements at the first level + std::vector _fields; + // The leaf node of schema elements + std::vector _physical_fields; + // Name to _fields, not all schema elements + std::unordered_map _name_to_field; + // Used in from_thrift, marking the next schema position that should be parsed + size_t _next_schema_pos; + // useful for parse_node_field to decide whether to convert byte_array to VARBINARY type + bool _enable_mapping_varbinary = false; + bool _enable_mapping_timestamp_tz = false; + +private: + void parse_physical_field(const tparquet::SchemaElement& physical_schema, bool is_nullable, + NativeFieldSchema* physical_field); + + Status parse_list_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* list_field); + + Status parse_map_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* map_field); + + Status parse_struct_field(const std::vector& t_schemas, + size_t curr_pos, NativeFieldSchema* struct_field); + + Status parse_group_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* group_field); + + Status parse_node_field(const std::vector& t_schemas, size_t curr_pos, + NativeFieldSchema* node_field); + + std::pair convert_to_doris_type(tparquet::LogicalType logicalType, + bool nullable); + std::pair convert_to_doris_type( + const tparquet::SchemaElement& physical_schema, bool nullable); + std::pair get_doris_type(const tparquet::SchemaElement& physical_schema, + bool nullable); + +public: + NativeFieldDescriptor() = default; + ~NativeFieldDescriptor() = default; + + /** + * Parse NativeFieldDescriptor from parquet thrift FileMetaData. + * @param t_schemas list of schema elements + */ + Status parse_from_thrift(const std::vector& t_schemas); + + int get_column_index(const std::string& column) const; + + /** + * Get the column(the first level schema element, maybe nested field) by index. + * @param index Column index in _fields + */ + const NativeFieldSchema* get_column(size_t index) const { return &_fields[index]; } + + /** + * Get the column(the first level schema element, maybe nested field) by name. + * @param name Column name + * @return NativeFieldSchema or nullptr if not exists + */ + NativeFieldSchema* get_column(const std::string& name) const; + + void get_column_names(std::unordered_set* names) const; + + std::string debug_string() const; + + int32_t size() const { return cast_set(_fields.size()); } + + const std::vector& get_fields_schema() const { return _fields; } + + /** + * Assign stable column IDs to schema fields. + * + * This uses an ORC-compatible encoding so that the results of + * create_column_ids() are consistent across formats. IDs start from 1 + * and are assigned in a pre-order traversal (parent before children). + * After calling this, each NativeFieldSchema will have column_id and + * max_column_id populated. + */ + void assign_ids(); + + const NativeFieldSchema* find_column_by_id(uint64_t column_id) const; + void set_enable_mapping_varbinary(bool enable) { _enable_mapping_varbinary = enable; } + void set_enable_mapping_timestamp_tz(bool enable) { _enable_mapping_timestamp_tz = enable; } +}; + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.cpp b/be/src/format_v2/parquet/native_schema_node.cpp new file mode 100644 index 00000000000000..7539fb45efb46d --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.cpp @@ -0,0 +1,127 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/native_schema_node.h" + +#include +#include + +#include "core/assert_cast.h" +#include "core/data_type/data_type_array.h" +#include "core/data_type/data_type_map.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/parquet_column_schema.h" +#include "util/string_util.h" + +namespace doris::format::parquet { + +void NativeStructSchemaNode::add_child(std::string table_name, std::string file_name, + std::shared_ptr node) { + _children.emplace(std::move(table_name), Child {std::move(file_name), std::move(node)}); +} + +void NativeStructSchemaNode::add_missing_child(std::string table_name) { + _children.emplace(std::move(table_name), Child {}); +} + +std::shared_ptr NativeStructSchemaNode::child( + const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? nullptr : it->second.node; +} + +std::string NativeStructSchemaNode::file_child_name(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it == _children.end() ? std::string {} : it->second.file_name; +} + +bool NativeStructSchemaNode::has_child(const std::string& table_name) const { + const auto it = _children.find(table_name); + return it != _children.end() && it->second.node != nullptr; +} + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result) { + if (projected_type == nullptr || result == nullptr) { + return Status::InvalidArgument("Native Parquet schema mapping input is null"); + } + const auto type = remove_nullable(projected_type); + switch (type->get_primitive_type()) { + case TYPE_STRUCT: { + if (file_schema.kind != ParquetColumnSchemaKind::STRUCT) { + return Status::Corruption("Parquet column {} is not a STRUCT", file_schema.name); + } + const auto* struct_type = assert_cast(type.get()); + std::map file_children; + for (const auto& child : file_schema.children) { + file_children.emplace(to_lower(child->name), child.get()); + } + auto node = std::make_shared(); + for (size_t i = 0; i < struct_type->get_elements().size(); ++i) { + const auto& table_name = struct_type->get_element_name(i); + // Native metadata keeps writer casing. Match normalized names while preserving the + // original file name used to address the physical child reader. + const auto child_it = file_children.find(to_lower(table_name)); + if (child_it == file_children.end()) { + node->add_missing_child(table_name); + continue; + } + std::shared_ptr child_node; + RETURN_IF_ERROR(build_native_schema_node(struct_type->get_element(i), *child_it->second, + &child_node)); + node->add_child(table_name, child_it->second->name, std::move(child_node)); + } + *result = std::move(node); + return Status::OK(); + } + case TYPE_ARRAY: { + if (file_schema.kind != ParquetColumnSchemaKind::LIST || file_schema.children.size() != 1) { + return Status::Corruption("Parquet column {} is not an ARRAY", file_schema.name); + } + const auto* array_type = assert_cast(type.get()); + std::shared_ptr element; + RETURN_IF_ERROR(build_native_schema_node(array_type->get_nested_type(), + *file_schema.children[0], &element)); + *result = std::make_shared(std::move(element)); + return Status::OK(); + } + case TYPE_MAP: { + if (file_schema.kind != ParquetColumnSchemaKind::MAP || file_schema.children.size() != 2) { + return Status::Corruption("Parquet column {} is not a MAP", file_schema.name); + } + const auto* map_type = assert_cast(type.get()); + std::shared_ptr key; + std::shared_ptr value; + RETURN_IF_ERROR( + build_native_schema_node(map_type->get_key_type(), *file_schema.children[0], &key)); + RETURN_IF_ERROR(build_native_schema_node(map_type->get_value_type(), + *file_schema.children[1], &value)); + *result = std::make_shared(std::move(key), std::move(value)); + return Status::OK(); + } + default: + if (file_schema.kind != ParquetColumnSchemaKind::PRIMITIVE) { + return Status::Corruption("Parquet column {} is not a scalar", file_schema.name); + } + *result = std::make_shared(); + return Status::OK(); + } +} + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/native_schema_node.h b/be/src/format_v2/parquet/native_schema_node.h new file mode 100644 index 00000000000000..25c42ad4b2c6d4 --- /dev/null +++ b/be/src/format_v2/parquet/native_schema_node.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "core/data_type/data_type.h" + +namespace doris::format::parquet { + +struct ParquetColumnSchema; + +// V2-owned semantic mapping consumed by the native decoder. It deliberately contains no V1 +// reader/schema-helper state, so FileScannerV2 can build the mapping from its native metadata tree. +class NativeSchemaNode { +public: + virtual ~NativeSchemaNode() = default; + + virtual std::shared_ptr child(const std::string&) const { return nullptr; } + virtual std::string file_child_name(const std::string&) const { return {}; } + virtual bool has_child(const std::string&) const { return false; } + virtual std::shared_ptr element() const { return nullptr; } + virtual std::shared_ptr key() const { return nullptr; } + virtual std::shared_ptr value() const { return nullptr; } +}; + +class NativeScalarSchemaNode final : public NativeSchemaNode {}; + +class NativeStructSchemaNode final : public NativeSchemaNode { +public: + void add_child(std::string table_name, std::string file_name, + std::shared_ptr node); + void add_missing_child(std::string table_name); + + std::shared_ptr child(const std::string& table_name) const override; + std::string file_child_name(const std::string& table_name) const override; + bool has_child(const std::string& table_name) const override; + +private: + struct Child { + std::string file_name; + std::shared_ptr node; + }; + std::map _children; +}; + +class NativeArraySchemaNode final : public NativeSchemaNode { +public: + explicit NativeArraySchemaNode(std::shared_ptr element) + : _element(std::move(element)) {} + std::shared_ptr element() const override { return _element; } + +private: + std::shared_ptr _element; +}; + +class NativeMapSchemaNode final : public NativeSchemaNode { +public: + NativeMapSchemaNode(std::shared_ptr key, + std::shared_ptr value) + : _key(std::move(key)), _value(std::move(value)) {} + std::shared_ptr key() const override { return _key; } + std::shared_ptr value() const override { return _value; } + +private: + std::shared_ptr _key; + std::shared_ptr _value; +}; + +Status build_native_schema_node(const DataTypePtr& projected_type, + const ParquetColumnSchema& file_schema, + std::shared_ptr* result); + +} // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index 59a315096cd92a..cefab47cf1cbc0 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -26,6 +26,7 @@ #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_type.h" namespace doris::format::parquet { @@ -470,6 +471,207 @@ Status build_node_schema(const ::parquet::SchemaDescriptor& schema, return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); } +ParquetTimeUnit native_time_unit(const tparquet::TimeUnit& unit) { + if (unit.__isset.MILLIS) { + return ParquetTimeUnit::MILLIS; + } + if (unit.__isset.MICROS) { + return ParquetTimeUnit::MICROS; + } + if (unit.__isset.NANOS) { + return ParquetTimeUnit::NANOS; + } + return ParquetTimeUnit::UNKNOWN; +} + +ParquetExtraTypeInfo native_time_extra(ParquetTimeUnit unit) { + switch (unit) { + case ParquetTimeUnit::MILLIS: + return ParquetExtraTypeInfo::UNIT_MS; + case ParquetTimeUnit::MICROS: + return ParquetExtraTypeInfo::UNIT_MICROS; + case ParquetTimeUnit::NANOS: + return ParquetExtraTypeInfo::UNIT_NS; + case ParquetTimeUnit::UNKNOWN: + default: + return ParquetExtraTypeInfo::NONE; + } +} + +void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDescriptor* result) { + DORIS_CHECK(result != nullptr); + const auto& schema = field.parquet_schema; + result->doris_type = field.data_type; + result->physical_type = static_cast<::parquet::Type::type>(field.physical_type); + result->fixed_length = schema.__isset.type_length ? schema.type_length : -1; + if (schema.__isset.converted_type) { + result->converted_type = static_cast<::parquet::ConvertedType::type>(schema.converted_type); + } + + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.DECIMAL) { + result->is_decimal = true; + result->decimal_precision = logical.DECIMAL.precision; + result->decimal_scale = logical.DECIMAL.scale; + } else if (logical.__isset.INTEGER) { + result->integer_bit_width = logical.INTEGER.bitWidth; + result->is_unsigned_integer = !logical.INTEGER.isSigned; + } else if (logical.__isset.TIME) { + result->time_unit = native_time_unit(logical.TIME.unit); + result->extra_type_info = native_time_extra(result->time_unit); + if (logical.TIME.isAdjustedToUTC) { + result->unsupported_reason = + "Parquet TIME with isAdjustedToUTC=true is not supported"; + } + } else if (logical.__isset.TIMESTAMP) { + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = logical.TIMESTAMP.isAdjustedToUTC; + result->time_unit = native_time_unit(logical.TIMESTAMP.unit); + result->extra_type_info = native_time_extra(result->time_unit); + } else if (logical.__isset.FLOAT16) { + result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; + } + } else if (schema.__isset.converted_type) { + switch (schema.converted_type) { + case tparquet::ConvertedType::DECIMAL: + result->is_decimal = true; + result->decimal_precision = schema.__isset.precision ? schema.precision : -1; + result->decimal_scale = schema.__isset.scale ? schema.scale : -1; + break; + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::UINT_8: + result->integer_bit_width = 8; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_8; + break; + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::UINT_16: + result->integer_bit_width = 16; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_16; + break; + case tparquet::ConvertedType::INT_32: + case tparquet::ConvertedType::UINT_32: + result->integer_bit_width = 32; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_32; + break; + case tparquet::ConvertedType::INT_64: + case tparquet::ConvertedType::UINT_64: + result->integer_bit_width = 64; + result->is_unsigned_integer = schema.converted_type == tparquet::ConvertedType::UINT_64; + break; + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + result->is_timestamp = true; + result->timestamp_is_adjusted_to_utc = true; + result->time_unit = schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS + ? ParquetTimeUnit::MILLIS + : ParquetTimeUnit::MICROS; + result->extra_type_info = native_time_extra(result->time_unit); + break; + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::TIME_MICROS: + result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; + break; + default: + break; + } + } + + if (result->is_decimal) { + switch (result->physical_type) { + case ::parquet::Type::INT32: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; + break; + case ::parquet::Type::INT64: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; + break; + case ::parquet::Type::BYTE_ARRAY: + case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; + break; + default: + break; + } + } else if (result->physical_type == ::parquet::Type::INT96) { + result->is_timestamp = true; + result->extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; + } + result->is_string_like = !result->is_decimal && + result->extra_type_info != ParquetExtraTypeInfo::FLOAT16 && + (result->physical_type == ::parquet::Type::BYTE_ARRAY || + result->physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); +} + +void propagate_native_max_levels(ParquetColumnSchema* schema) { + DORIS_CHECK(schema != nullptr); + for (const auto& child : schema->children) { + DORIS_CHECK(child != nullptr); + propagate_native_max_levels(child.get()); + schema->max_definition_level = + std::max(schema->max_definition_level, child->max_definition_level); + schema->max_repetition_level = + std::max(schema->max_repetition_level, child->max_repetition_level); + } +} + +std::unique_ptr build_native_node_schema(const NativeFieldSchema& field, + int32_t local_id) { + auto result = std::make_unique(); + result->local_id = local_id; + result->parquet_field_id = field.field_id; + result->name = field.name; + result->type = field.data_type; + result->definition_level = field.definition_level; + result->repetition_level = field.repetition_level; + result->max_definition_level = field.definition_level; + result->max_repetition_level = field.repetition_level; + result->nullable_definition_level = field.data_type != nullptr && field.data_type->is_nullable() + ? field.definition_level - field.repetition_level + : 0; + result->repeated_ancestor_definition_level = field.repeated_parent_def_level; + result->repeated_repetition_level = field.repetition_level; + + const auto primitive_type = remove_nullable(field.data_type)->get_primitive_type(); + if (field.children.empty()) { + result->kind = ParquetColumnSchemaKind::PRIMITIVE; + result->leaf_column_id = field.physical_column_index; + fill_native_type_descriptor(field, &result->type_descriptor); + return result; + } + if (primitive_type == TYPE_ARRAY) { + result->kind = ParquetColumnSchemaKind::LIST; + } else if (primitive_type == TYPE_MAP) { + result->kind = ParquetColumnSchemaKind::MAP; + } else { + result->kind = ParquetColumnSchemaKind::STRUCT; + } + result->children.reserve(field.children.size()); + for (size_t child_idx = 0; child_idx < field.children.size(); ++child_idx) { + result->children.push_back( + build_native_node_schema(field.children[child_idx], cast_set(child_idx))); + } + propagate_native_max_levels(result.get()); + return result; +} + +Status validate_native_node_schema(const NativeFieldSchema& field) { + if (field.children.empty()) { + ParquetTypeDescriptor descriptor; + fill_native_type_descriptor(field, &descriptor); + if (!descriptor.unsupported_reason.empty()) { + // Native metadata must enforce the same logical-type contract used by scan planning; + // otherwise unsupported files reach the decoder only after their schema is accepted. + return Status::NotSupported("Unsupported parquet column '{}': {}", field.name, + descriptor.unsupported_reason); + } + return Status::OK(); + } + for (const auto& child : field.children) { + RETURN_IF_ERROR(validate_native_node_schema(child)); + } + return Status::OK(); +} + } // namespace Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, @@ -494,4 +696,22 @@ Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, return Status::OK(); } +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, + std::vector>* fields) { + if (fields == nullptr) { + return Status::InvalidArgument("fields is null"); + } + fields->clear(); + const auto& native_fields = schema.get_fields_schema(); + fields->reserve(native_fields.size()); + for (size_t field_idx = 0; field_idx < native_fields.size(); ++field_idx) { + RETURN_IF_ERROR(validate_native_node_schema(native_fields[field_idx])); + // The scan projection and native readers must share one tree; rebuilding wrappers through + // Arrow changes legacy LIST/STRUCT boundaries and makes valid nested values look absent. + fields->push_back( + build_native_node_schema(native_fields[field_idx], cast_set(field_idx))); + } + return Status::OK(); +} + } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index ecacdfe1d97c4c..76ae58b24d2c0a 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -30,6 +30,8 @@ class SchemaDescriptor; namespace doris::format::parquet { +class NativeFieldDescriptor; + enum class ParquetColumnSchemaKind { PRIMITIVE, // physical primitive leaf STRUCT, // Parquet group with STRUCT semantics @@ -77,4 +79,7 @@ struct ParquetColumnSchema { Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, std::vector>* fields); +Status build_parquet_column_schema(const NativeFieldDescriptor& schema, + std::vector>* fields); + } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 8c9caddfde12ec..03bb38daf095a9 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -52,17 +52,13 @@ namespace doris::format::parquet { -NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size, - std::vector serialized_metadata) - : _metadata(std::move(metadata)), - _parsed_size(parsed_size), - _serialized_metadata(std::move(serialized_metadata)) { +NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) + : _metadata(std::move(metadata)), _parsed_size(parsed_size) { ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); } NativeParquetMetadata::~NativeParquetMetadata() { - ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size() + - _arrow_metadata_mem_size); + ExecEnv::GetInstance()->parquet_meta_tracker()->release(get_mem_size()); } Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, @@ -76,38 +72,6 @@ Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, return Status::OK(); } -Status NativeParquetMetadata::get_arrow_metadata( - std::shared_ptr<::parquet::FileMetaData>* metadata) const { - DORIS_CHECK(metadata != nullptr); - std::lock_guard lock(_arrow_metadata_mutex); - if (_arrow_metadata == nullptr) { - try { - uint32_t serialized_size = cast_set(_serialized_metadata.size()); - auto parsed = - ::parquet::FileMetaData::Make(_serialized_metadata.data(), &serialized_size, - ::parquet::default_reader_properties()); - if (static_cast(serialized_size) != _serialized_metadata.size()) { - return Status::Corruption("Arrow consumed {} of {} Parquet footer bytes", - serialized_size, _serialized_metadata.size()); - } - _arrow_metadata_mem_size = parsed->size(); - ExecEnv::GetInstance()->parquet_meta_tracker()->consume(_arrow_metadata_mem_size); - _arrow_metadata = std::move(parsed); - } catch (const ::parquet::ParquetException& e) { - return Status::Corruption("Failed to adapt v2 Parquet metadata: {}", e.what()); - } catch (const std::exception& e) { - return Status::InternalError("Failed to adapt v2 Parquet metadata: {}", e.what()); - } - } - *metadata = _arrow_metadata; - return Status::OK(); -} - -size_t NativeParquetMetadata::arrow_metadata_mem_size() const { - std::lock_guard lock(_arrow_metadata_mutex); - return _arrow_metadata_mem_size; -} - namespace detail { namespace { @@ -179,6 +143,19 @@ size_t ParquetPageCacheRangeDirectory::size() const { return _indexes.size(); } +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length) { + if (offset < 0 || length <= 0 || length > MAX_SERIALIZED_PARQUET_INDEX_BYTES || + static_cast(offset) > file_size) { + return false; + } + return static_cast(length) <= file_size - static_cast(offset); +} + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end) { + return span_offset >= 0 && span_end >= span_offset && + span_end - span_offset <= MAX_SERIALIZED_PARQUET_INDEX_BYTES; +} + std::vector plan_page_cache_range_read( int64_t position, int64_t nbytes, const std::vector& cached_ranges) { if (position < 0 || nbytes <= 0) { @@ -342,8 +319,8 @@ Status parse_native_parquet_footer(io::FileReaderSPtr file, tparquet::FileMetaData thrift_metadata; RETURN_IF_ERROR(deserialize_thrift_msg(serialized_metadata.data(), &thrift_size, true, &thrift_metadata)); - auto parsed = std::make_unique( - std::move(thrift_metadata), serialized_size, std::move(serialized_metadata)); + auto parsed = + std::make_unique(std::move(thrift_metadata), serialized_size); RETURN_IF_ERROR(parsed->init_schema(enable_mapping_varbinary, enable_mapping_timestamp_tz)); *footer_size = V2_PARQUET_FOOTER_SIZE + serialized_size; *metadata = std::move(parsed); @@ -371,8 +348,8 @@ std::string build_page_cache_file_key(const io::FileReader& file_reader, class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { public: - DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) + [[maybe_unused]] DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, + bool enable_page_cache, std::string page_cache_file_key) : _file_reader(std::move(file_reader)), _base_file_reader(_file_reader), _io_ctx(io_ctx), @@ -473,13 +450,13 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { return buffer; } - void register_page_cache_ranges(std::vector ranges) { + [[maybe_unused]] void register_page_cache_ranges(std::vector ranges) { std::lock_guard lock(_page_cache_mutex); _page_cache_ranges = std::move(ranges); } - void prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { + [[maybe_unused]] void prefetch_ranges(const std::vector& ranges, + const io::IOContext* io_ctx) { auto cached_reader = cached_remote_file_reader(); if (cached_reader == nullptr) { return; @@ -494,9 +471,9 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { } } - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { + [[maybe_unused]] bool set_random_access_ranges(const std::vector& ranges, + size_t avg_io_size, RuntimeProfile* profile, + int64_t merge_read_slice_size) { reset_active_file_reader(); const auto valid_ranges = detail::valid_prefetch_ranges(ranges); if (!detail::should_use_merge_range_reader( @@ -525,9 +502,9 @@ class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { return true; } - void reset_random_access_ranges() { reset_active_file_reader(); } + [[maybe_unused]] void reset_random_access_ranges() { reset_active_file_reader(); } - ParquetPageCacheStats page_cache_stats() const { + [[maybe_unused]] ParquetPageCacheStats page_cache_stats() const { std::lock_guard lock(_page_cache_mutex); return _page_cache_stats; } @@ -752,38 +729,8 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont auto page_cache_file_key = build_page_cache_file_key(*native_file, file_description); native_page_cache_enabled = enable_page_cache && !page_cache_file_key.empty(); - // Native and Arrow readers must use the same FileDescription-derived immutable identity. + // Native page readers use the FileDescription-derived immutable identity directly. native_page_cache_file_key = page_cache_file_key; - arrow_file = std::make_shared(native_file, io_ctx, enable_page_cache, - std::move(page_cache_file_key)); - try { - const int64_t adapter_start_ns = MonotonicNanos(); - std::shared_ptr<::parquet::FileMetaData> arrow_metadata; - RETURN_IF_ERROR(native_metadata->get_arrow_metadata(&arrow_metadata)); - this->file_reader = ::parquet::ParquetFileReader::Open( - arrow_file, ::parquet::default_reader_properties(), std::move(arrow_metadata)); - metadata = this->file_reader->metadata(); - schema = metadata != nullptr ? metadata->schema() : nullptr; - arrow_metadata_adapter_time += MonotonicNanos() - adapter_start_ns; - arrow_metadata_adapter_bytes = - static_cast(native_metadata->arrow_metadata_mem_size()); - } catch (const ::parquet::ParquetException& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::Corruption("Failed to open parquet file: {}", e.what()); - } catch (const std::exception& e) { - if (io_ctx != nullptr && io_ctx->should_stop && - std::string_view(e.what()).find("stop") != std::string_view::npos) { - return Status::EndOfFile("stop"); - } - return Status::InternalError("Failed to open parquet file: {}", e.what()); - } - - if (metadata == nullptr || schema == nullptr) { - return Status::Corruption("Failed to read parquet metadata"); - } return Status::OK(); } @@ -817,9 +764,8 @@ Status ParquetFileContext::load_native_offset_indexes( } const int64_t index_offset = column_chunk.offset_index_offset; const int64_t index_length = column_chunk.offset_index_length; - if (index_offset < 0 || index_length <= 0 || index_offset > native_file->size() || - index_length > native_file->size() - index_offset || - index_length > std::numeric_limits::max()) { + if (!detail::is_serialized_index_range_safe(native_file->size(), index_offset, + index_length)) { // OffsetIndex is optional. A malformed range must not allocate from untrusted // footer values or redirect the native reader outside the file. continue; @@ -893,12 +839,7 @@ Status ParquetFileContext::load_native_page_indexes( std::unordered_map pending_indexes; auto valid_index_range = [&](int64_t offset, int64_t length) { - if (offset < 0 || length <= 0 || offset > native_file->size() || - length > native_file->size() - offset || - length > std::numeric_limits::max()) { - return false; - } - return true; + return detail::is_serialized_index_range_safe(native_file->size(), offset, length); }; for (const int leaf_column_id : leaf_column_ids) { @@ -936,6 +877,12 @@ Status ParquetFileContext::load_native_page_indexes( } const int64_t span_offset = (*ranges)[range_begin].offset; + if (!detail::is_serialized_index_span_safe(span_offset, span_end)) { + // Optional indexes share one allocation per contiguous footer block. Skipping the + // whole block prevents many individually small ranges from bypassing the budget. + range_begin = range_end; + continue; + } const int64_t span_length = span_end - span_offset; std::vector serialized(static_cast(span_length)); Slice slice(serialized.data(), serialized.size()); @@ -1001,23 +948,36 @@ Status ParquetFileContext::load_native_page_indexes( } void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get()) - ->register_page_cache_ranges(std::move(ranges)); + // Native column readers register exact page payloads themselves; retaining a second range map + // would recreate the removed Arrow metadata adapter's cache path. + (void)ranges; } void ParquetFileContext::prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx) { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->prefetch_ranges(ranges, io_ctx); + io::FileReaderSPtr reader = native_file; + if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { + reader = tracing_reader->inner_reader(); + } + auto cached_reader = std::dynamic_pointer_cast(reader); + if (cached_reader == nullptr) { + return; + } + const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : native_io_ctx; + for (const auto& range : detail::valid_prefetch_ranges(ranges)) { + cached_reader->prefetch_range(cast_set(range.offset), cast_set(range.size), + prefetch_io_ctx); + } } bool ParquetFileContext::set_random_access_ranges(const std::vector& ranges, size_t avg_io_size, RuntimeProfile* profile, int64_t merge_read_slice_size) { - DORIS_CHECK(arrow_file != nullptr); - return static_cast(arrow_file.get()) - ->set_random_access_ranges(ranges, avg_io_size, profile, merge_read_slice_size); + (void)ranges; + (void)avg_io_size; + (void)profile; + (void)merge_read_slice_size; + return false; } bool ParquetFileContext::set_native_random_access_ranges( @@ -1045,8 +1005,6 @@ bool ParquetFileContext::set_native_random_access_ranges( } void ParquetFileContext::reset_random_access_ranges() { - DORIS_CHECK(arrow_file != nullptr); - static_cast(arrow_file.get())->reset_random_access_ranges(); if (native_row_group_file != nullptr && native_row_group_file != native_file) { native_row_group_file->collect_profile_before_close(); } @@ -1054,10 +1012,7 @@ void ParquetFileContext::reset_random_access_ranges() { } ParquetPageCacheStats ParquetFileContext::page_cache_stats() const { - if (arrow_file == nullptr) { - return {}; - } - return static_cast(arrow_file.get())->page_cache_stats(); + return {}; } Status ParquetFileContext::close() { @@ -1065,17 +1020,6 @@ Status ParquetFileContext::close() { native_row_group_file->collect_profile_before_close(); } native_row_group_file.reset(); - if (file_reader != nullptr) { - try { - file_reader->Close(); - } catch (const std::exception&) { - } - } - if (arrow_file != nullptr) { - static_cast(arrow_status_to_doris_status(arrow_file->Close())); - } - file_reader.reset(); - arrow_file.reset(); native_metadata = nullptr; native_metadata_owner.reset(); native_meta_cache_handle = {}; diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 5e62becfb81e29..46854179b2915c 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -30,7 +30,7 @@ #include #include "common/status.h" -#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/native_schema_desc.h" #include "io/fs/file_reader.h" #include "util/obj_lru_cache.h" @@ -47,29 +47,22 @@ namespace doris::format::parquet { struct NativeParquetPageIndex; -// V2-owned footer/schema object. Keeping this adapter here prevents native scanning requirements -// from changing the established v1 FileMetaData cache value or parser behavior. +// V2-owned footer/schema tree. Production planning and decoding consume this object directly; +// Arrow metadata is intentionally not materialized from the serialized footer. class NativeParquetMetadata { public: - NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size, - std::vector serialized_metadata); + NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size); ~NativeParquetMetadata(); Status init_schema(bool enable_mapping_varbinary, bool enable_mapping_timestamp_tz); const tparquet::FileMetaData& to_thrift() const { return _metadata; } - const FieldDescriptor& schema() const { return _schema; } - Status get_arrow_metadata(std::shared_ptr<::parquet::FileMetaData>* metadata) const; - size_t arrow_metadata_mem_size() const; - size_t get_mem_size() const { return _parsed_size + _serialized_metadata.size(); } + const NativeFieldDescriptor& schema() const { return _schema; } + size_t get_mem_size() const { return _parsed_size; } private: tparquet::FileMetaData _metadata; - FieldDescriptor _schema; + NativeFieldDescriptor _schema; size_t _parsed_size = 0; - std::vector _serialized_metadata; - mutable std::mutex _arrow_metadata_mutex; - mutable std::shared_ptr<::parquet::FileMetaData> _arrow_metadata; - mutable size_t _arrow_metadata_mem_size = 0; }; struct ParquetPageCacheRange { @@ -134,6 +127,12 @@ class ParquetPageCacheRangeDirectory { std::unordered_map> _indexes; }; +inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; + +bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length); + +bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); + // Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of // previously cached entries. // StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The @@ -179,9 +178,8 @@ bool should_stage_small_http_file(std::string_view path, size_t file_size, } // namespace detail struct ParquetFileContext { - // The native data-page path reads from Doris' FileReader directly. Keep this handle separate - // from the Arrow RandomAccessFile used by the metadata/index migration path so opening Arrow - // metadata never transfers ownership away from the native reader. + // Native metadata, index, and data-page paths share Doris' FileReader without transferring + // ownership to an external metadata tree. io::FileReaderSPtr native_file; // Row-group-scoped view of native_file. Small projected chunks use the same // MergeRangeFileReader policy as v1; large chunks and in-memory files keep native_file. @@ -194,16 +192,9 @@ struct ParquetFileContext { ObjLRUCache::CacheHandle native_meta_cache_handle; int64_t native_footer_read_calls = 0; int64_t native_footer_cache_hits = 0; - int64_t arrow_metadata_adapter_time = 0; - int64_t arrow_metadata_adapter_bytes = 0; bool native_page_cache_enabled = false; std::string native_page_cache_file_key; - std::shared_ptr arrow_file; // Arrow wrapper for Doris FileReader - std::unique_ptr<::parquet::ParquetFileReader> file_reader; // Arrow Parquet file parser - std::shared_ptr<::parquet::FileMetaData> metadata; // footer metadata (RowGroup information) - const ::parquet::SchemaDescriptor* schema = nullptr; // physical leaf column schema - Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, bool enable_mapping_timestamp_tz = false); @@ -215,16 +206,14 @@ struct ParquetFileContext { std::unordered_map* page_indexes, int64_t* read_time = nullptr, int64_t* parse_time = nullptr) const; - // Register ranges for the remaining Arrow metadata/index adapter. Native data pages use the - // v1-compatible page cache owned by BufferedFileStreamReader instead. + // Retained as a compatibility hook for callers; native readers admit exact page payloads. void register_page_cache_ranges(std::vector ranges); // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Switch the active reader used by Arrow metadata/index ReadAt() to MergeRangeFileReader when - // projected chunks are small random IOs. Native page decoding is intentionally independent. + // Deprecated adapter hook. Native readers use set_native_random_access_ranges(). bool set_random_access_ranges(const std::vector& ranges, size_t avg_io_size, RuntimeProfile* profile, int64_t merge_read_slice_size); @@ -237,8 +226,7 @@ struct ParquetFileContext { const io::FileReaderSPtr& native_data_file() const { return native_row_group_file != nullptr ? native_row_group_file : native_file; } - // Restore both Arrow and native ReadAt() to the base Doris file reader and flush active - // merge-reader counters. Row-group setup uses this before dictionary-page probes. + // Restore native ReadAt() to the base Doris file reader and flush merge-reader counters. void reset_random_access_ranges(); ParquetPageCacheStats page_cache_stats() const; Status close(); diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 3c5de0a0a494b9..f197c0ec188858 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -60,8 +60,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); selected_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "SelectedRows", TUnit::UNIT, parquet_profile, 1); - rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "RowsFilteredByConjunct", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + // Keep every Parquet scan metric below the format node: profile consumers extract that + // subtree and otherwise silently lose this counter even though filtering happened. + rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByConjunct", + TUnit::UNIT, parquet_profile, 1); total_batches = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "TotalBatches", TUnit::UNIT, parquet_profile, 1); dense_batches = @@ -106,10 +108,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); parse_meta_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseMetaTime", parquet_profile, 1); - arrow_metadata_adapter_time = - ADD_CHILD_TIMER_WITH_LEVEL(profile, "ArrowMetadataAdapterTime", parquet_profile, 1); - arrow_metadata_adapter_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( - profile, "ArrowMetadataAdapterBytes", TUnit::BYTES, parquet_profile, 1); parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); file_reader_create_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 28b63eacd24ab5..d17939e70e7a41 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -162,8 +162,6 @@ struct ParquetProfile { RuntimeProfile::Counter* column_read_time = nullptr; RuntimeProfile::Counter* parse_meta_time = nullptr; - RuntimeProfile::Counter* arrow_metadata_adapter_time = nullptr; - RuntimeProfile::Counter* arrow_metadata_adapter_bytes = nullptr; RuntimeProfile::Counter* parse_footer_time = nullptr; RuntimeProfile::Counter* file_reader_create_time = nullptr; RuntimeProfile::Counter* open_file_num = nullptr; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 21828e2a601891..99dbfa1bd5b294 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -177,7 +177,7 @@ Status validate_requested_columns_supported( } std::vector build_page_cache_ranges( - const ::parquet::FileMetaData& metadata, + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { std::unordered_set leaf_column_ids; @@ -185,14 +185,15 @@ std::vector build_page_cache_ranges( std::vector ranges; ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); for (const auto& row_group_plan_item : row_group_plan.row_groups) { - auto row_group_metadata = metadata.RowGroup(row_group_plan_item.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); + const auto& row_group_metadata = metadata.row_groups[row_group_plan_item.row_group_id]; for (const auto leaf_column_id : leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_chunk_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); + DORIS_CHECK(leaf_column_id >= 0 && + leaf_column_id < static_cast(row_group_metadata.columns.size())); + const auto& column_metadata = row_group_metadata.columns[leaf_column_id].meta_data; + const int64_t offset = column_metadata.__isset.dictionary_page_offset + ? column_metadata.dictionary_page_offset + : column_metadata.data_page_offset; + const int64_t size = column_metadata.total_compressed_size; DORIS_CHECK(offset >= 0); DORIS_CHECK(size >= 0); if (size > 0) { @@ -345,8 +346,7 @@ static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schem } static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { - DORIS_CHECK(column_schema.descriptor != nullptr); - switch (column_schema.descriptor->physical_type()) { + switch (column_schema.type_descriptor.physical_type) { case ::parquet::Type::BYTE_ARRAY: case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be @@ -436,17 +436,13 @@ Status ParquetReader::init(RuntimeState* state) { _state->file_context.native_footer_read_calls); COUNTER_UPDATE(_parquet_profile.file_footer_hit_cache, _state->file_context.native_footer_cache_hits); - COUNTER_UPDATE(_parquet_profile.arrow_metadata_adapter_time, - _state->file_context.arrow_metadata_adapter_time); - COUNTER_UPDATE(_parquet_profile.arrow_metadata_adapter_bytes, - _state->file_context.arrow_metadata_adapter_bytes); } // Build file schema from parquet metadata. // A file reader may expose raw file identifiers, such as Parquet field_id, through ColumnDefinition::identifier { SCOPED_TIMER(_parquet_profile.parse_meta_time); - RETURN_IF_ERROR( - build_parquet_column_schema(*_state->file_context.schema, &_state->file_schema)); + RETURN_IF_ERROR(build_parquet_column_schema(_state->file_context.native_metadata->schema(), + &_state->file_schema)); if (_enable_mapping_timestamp_tz) { for (auto& column_schema : _state->file_schema) { apply_timestamp_tz_mapping(column_schema.get()); @@ -469,7 +465,7 @@ Status ParquetReader::get_schema(std::vector* file_sch return Status::InvalidArgument("file_schema is null"); } file_schema->clear(); - if (_state == nullptr || _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } @@ -493,8 +489,7 @@ std::unique_ptr ParquetReader::create_column_mapper( Status ParquetReader::open(std::shared_ptr request) { SCOPED_TIMER(_parquet_profile.total_time); - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } auto request_snapshot = request; @@ -547,16 +542,16 @@ Status ParquetReader::open(std::shared_ptr request) { scan_range.file_size = _file_description->file_size; // Get selected ranges in row groups according to metadata (Row-Group level index and Page Index including Zonemap, Dictionary, Bloom Filter). RETURN_IF_ERROR(plan_parquet_row_groups( - *_state->file_context.metadata, _state->file_context.file_reader.get(), - _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, - &row_group_plan, _state->timezone, _state->runtime_state, &_state->file_context)); + *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, + scan_range, _state->enable_bloom_filter, &row_group_plan, _state->timezone, + _state->runtime_state, &_state->file_context)); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } if (_state->enable_page_cache) { _state->file_context.register_page_cache_ranges( - build_page_cache_ranges(*_state->file_context.metadata, _state->file_schema, - *request_snapshot, row_group_plan)); + build_page_cache_ranges(_state->file_context.native_metadata->to_thrift(), + _state->file_schema, *request_snapshot, row_group_plan)); } _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); @@ -569,8 +564,7 @@ Status ParquetReader::open(std::shared_ptr request) { Status ParquetReader::get_block(Block* file_block, size_t* rows, bool* eof) { SCOPED_TIMER(_parquet_profile.total_time); - if (_state == nullptr || _state->file_context.file_reader == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } *rows = 0; @@ -671,8 +665,7 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r format::FileAggregateResult* result) { SCOPED_TIMER(_parquet_profile.total_time); DORIS_CHECK(result != nullptr); - if (_state == nullptr || _state->file_context.metadata == nullptr || - _state->file_context.schema == nullptr) { + if (_state == nullptr || _state->file_context.native_metadata == nullptr) { return Status::Uninitialized("ParquetReader is not open"); } if (_should_stop()) { @@ -701,10 +694,9 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r // Aggregate row count in all selected row groups. For MIN/MAX aggregate, this is used to determine whether there is no row group selected. for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - result->count += row_group_metadata->num_rows(); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + result->count += row_group_metadata.num_rows; } if (request.agg_type == TPushAggOp::type::COUNT) { if (request.columns.empty()) { @@ -787,13 +779,18 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r auto& aggregate_column = result->columns[request_column_idx]; aggregate_column.projection = request.columns[request_column_idx].projection; for (const auto& row_group_plan : _state->scan_plan.row_groups) { - auto row_group_metadata = - _state->file_context.metadata->RowGroup(row_group_plan.row_group_id); - DORIS_CHECK(row_group_metadata != nullptr); - auto column_chunk = row_group_metadata->ColumnChunk(leaf_schema->leaf_column_id); - DORIS_CHECK(column_chunk != nullptr); + const auto& row_group_metadata = _state->file_context.native_metadata->to_thrift() + .row_groups[row_group_plan.row_group_id]; + DORIS_CHECK(leaf_schema->leaf_column_id >= 0 && + leaf_schema->leaf_column_id < + static_cast(row_group_metadata.columns.size())); + const auto& column_chunk = row_group_metadata.columns[leaf_schema->leaf_column_id]; + DORIS_CHECK(column_chunk.__isset.meta_data); + const auto& column_metadata = column_chunk.meta_data; const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( - *leaf_schema, column_chunk->statistics(), _state->timezone); + *leaf_schema, + column_metadata.__isset.statistics ? &column_metadata.statistics : nullptr, + column_metadata.num_values, _state->timezone); if (!statistics.has_min_max) { return Status::NotSupported("Missing parquet min/max statistics for column {}", leaf_schema->name); diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 8a38cc4a1c457c..67e9cd636604c5 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -106,26 +106,27 @@ int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadat : cast_set(column_metadata.data_page_offset()); } -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; +bool is_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; } -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; +bool is_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; } -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; +bool is_data_page_type(tparquet::PageType::type page_type) { + return page_type == tparquet::PageType::DATA_PAGE || + page_type == tparquet::PageType::DATA_PAGE_V2; } -bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { +bool is_fully_dictionary_encoded_chunk(const tparquet::ColumnMetaData& column_metadata) { + if (!column_metadata.__isset.dictionary_page_offset || + column_metadata.dictionary_page_offset < 0) { return false; } - const auto& encoding_stats = column_metadata.encoding_stats(); + const auto& encoding_stats = column_metadata.encoding_stats; if (!encoding_stats.empty()) { bool has_dictionary_data_page = false; for (const auto& encoding_stat : encoding_stats) { @@ -141,7 +142,7 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { + for (const auto encoding : column_metadata.encodings) { if (is_dictionary_data_encoding(encoding)) { has_dictionary_encoding = true; continue; @@ -154,14 +155,13 @@ bool is_fully_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& col } bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr || + const tparquet::ColumnMetaData& column_metadata) { + if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || column_schema.type == nullptr || column_schema.max_repetition_level > 0) { return false; } if (!column_schema.type_descriptor.is_string_like || - column_metadata.type() != ::parquet::Type::BYTE_ARRAY) { + column_metadata.type != tparquet::Type::BYTE_ARRAY) { return false; } // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact @@ -273,7 +273,7 @@ void materialize_count_star_placeholders(const format::FileScanRequest& request, } std::vector build_row_group_prefetch_ranges( - const ::parquet::FileMetaData& metadata, + const tparquet::FileMetaData& metadata, const std::vector>& file_schema, const std::vector& scan_columns, int row_group_idx) { std::unordered_set leaf_column_ids; @@ -291,19 +291,25 @@ std::vector build_row_group_prefetch_ranges( collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); } - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); + DORIS_CHECK(row_group_idx >= 0 && row_group_idx < static_cast(metadata.row_groups.size())); + const auto& row_group_metadata = metadata.row_groups[row_group_idx]; std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); std::ranges::sort(ordered_leaf_column_ids); std::vector ranges; ranges.reserve(ordered_leaf_column_ids.size()); for (const auto leaf_column_id : ordered_leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && leaf_column_id < row_group_metadata->num_columns()); - auto column_metadata = row_group_metadata->ColumnChunk(leaf_column_id); - DORIS_CHECK(column_metadata != nullptr); - const int64_t offset = column_start_offset(*column_metadata); - const int64_t size = column_metadata->total_compressed_size(); + DORIS_CHECK(leaf_column_id >= 0 && + leaf_column_id < static_cast(row_group_metadata.columns.size())); + const auto& chunk = row_group_metadata.columns[leaf_column_id]; + if (!chunk.__isset.meta_data) { + continue; + } + const auto& column_metadata = chunk.meta_data; + const int64_t offset = column_metadata.__isset.dictionary_page_offset + ? column_metadata.dictionary_page_offset + : column_metadata.data_page_offset; + const int64_t size = column_metadata.total_compressed_size; DORIS_CHECK(offset >= 0); if (size > 0) { ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); @@ -459,6 +465,136 @@ Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, namespace { +int64_t native_column_start_offset(const tparquet::ColumnMetaData& column_metadata) { + return column_metadata.__isset.dictionary_page_offset ? column_metadata.dictionary_page_offset + : column_metadata.data_page_offset; +} + +bool is_native_row_group_outside_range(const tparquet::RowGroup& row_group, + const ParquetScanRange& scan_range) { + if (scan_range.size < 0) { + return false; + } + const int64_t range_start = scan_range.start_offset; + const int64_t range_end = range_start + scan_range.size; + DORIS_CHECK(range_start >= 0 && range_end >= range_start); + if (range_start == 0 && (scan_range.file_size < 0 || range_end >= scan_range.file_size)) { + return false; + } + if (row_group.columns.empty() || !row_group.columns.front().__isset.meta_data || + !row_group.columns.back().__isset.meta_data) { + return false; + } + const auto& first = row_group.columns.front().meta_data; + const auto& last = row_group.columns.back().meta_data; + const int64_t group_start = native_column_start_offset(first); + const int64_t group_end = native_column_start_offset(last) + last.total_compressed_size; + const int64_t group_mid = group_start + (group_end - group_start) / 2; + return group_mid < range_start || group_mid >= range_end; +} + +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups) { + DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); + row_group_first_rows->assign(metadata.row_groups.size(), 0); + selected_row_groups->clear(); + int64_t next_first_row = 0; + for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { + (*row_group_first_rows)[row_group_idx] = next_first_row; + const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Invalid negative row count in parquet row group {}", + row_group_idx); + } + next_first_row += row_group.num_rows; + if (!is_native_row_group_outside_range(row_group, scan_range)) { + selected_row_groups->push_back(cast_set(row_group_idx)); + } + } + return Status::OK(); +} + +Status build_native_row_group_read_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector& selected_row_groups, + const std::vector& row_group_first_rows, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + const auto& thrift = metadata.to_thrift(); + std::unordered_set requested_leaf_ids; + for (const auto& projection : request_scan_columns(request)) { + const auto local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + continue; + } + collect_projected_leaf_column_ids(*file_schema[local_id], projection, &requested_leaf_ids); + } + plan->row_groups.reserve(selected_row_groups.size()); + for (const int row_group_idx : selected_row_groups) { + const auto& row_group = thrift.row_groups[row_group_idx]; + if (row_group.num_rows == 0) { + continue; + } + RowGroupReadPlan row_group_plan; + row_group_plan.row_group_id = row_group_idx; + row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; + row_group_plan.row_group_rows = row_group.num_rows; + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_idx, requested_leaf_ids, &page_indexes, + &plan->pruning_stats.read_page_index_time, + &plan->pruning_stats.parse_page_index_time)); + } + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + page_indexes, file_schema, request, row_group.num_rows, + &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, + &plan->pruning_stats, timezone, runtime_state)); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan.offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); + } + if (row_group_plan.selected_ranges.empty()) { + continue; + } + plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + plan->row_groups.push_back(std::move(row_group_plan)); + } + return Status::OK(); +} + +} // namespace + +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, + const ParquetScanRange& scan_range, bool enable_bloom_filter, + RowGroupScanPlan* plan, const cctz::time_zone* timezone, + const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + plan->row_groups.clear(); + plan->pruning_stats = {}; + std::vector row_group_first_rows; + std::vector scan_range_selected; + RETURN_IF_ERROR(select_native_row_groups_by_scan_range( + metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); + std::vector metadata_selected; + RETURN_IF_ERROR(select_row_groups_by_metadata( + metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, + enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context)); + RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, + metadata_selected, row_group_first_rows, plan, + timezone, runtime_state, file_context)); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} + +namespace { + using DictionaryResidualConjunct = std::pair; using DictionaryResidualConjuncts = std::vector; @@ -871,14 +1007,14 @@ Status ParquetScanScheduler::open_next_row_group( } RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; const int row_group_idx = row_group_plan.row_group_id; - // Row-level dictionary filters still use the migration metadata/page probe. Native data-page - // readers below do not construct an Arrow RowGroupReader or RecordReader. + // Dictionary probes and data-page readers share the native metadata tree. Reset the previous + // row-group merge reader before probing because dictionary-page offsets are not scan ordered. file_context.reset_random_access_ranges(); _current_merge_range_active = false; - auto row_group_metadata = file_context.metadata->RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - _current_row_group_rows = row_group_metadata->num_rows(); + const auto& row_group_metadata = + file_context.native_metadata->to_thrift().row_groups[row_group_idx]; + _current_row_group_rows = row_group_metadata.num_rows; DORIS_CHECK(_current_row_group_rows == row_group_plan.row_group_rows); DORIS_CHECK(_current_row_group_rows > 0); _current_row_group_id = row_group_idx; @@ -914,13 +1050,14 @@ Status ParquetScanScheduler::open_next_row_group( _current_non_predicate_columns.clear(); _current_dictionary_filters.clear(); RETURN_IF_ERROR(prepare_current_dictionary_filters(file_context, file_schema, request, - row_group_idx, *row_group_metadata)); + row_group_idx, row_group_metadata)); // Dictionary probing is complete, so the native data-page readers can now share the same // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy // materialization for wide schemas. - const auto native_ranges = build_row_group_prefetch_ranges( - *file_context.metadata, file_schema, request_scan_columns(request), row_group_idx); + const auto native_ranges = + build_row_group_prefetch_ranges(file_context.native_metadata->to_thrift(), file_schema, + request_scan_columns(request), row_group_idx); _current_merge_range_active = file_context.set_native_random_access_ranges( native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, _merge_read_slice_size); @@ -1179,7 +1316,7 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata) { + const tparquet::RowGroup& row_group_metadata) { _current_dictionary_filters.clear(); _current_dictionary_residual_conjuncts.clear(); if (request.conjuncts.empty()) { @@ -1216,13 +1353,13 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( const auto& column_schema = file_schema[local_id]; DORIS_CHECK(column_schema != nullptr); if (column_schema->leaf_column_id < 0 || - column_schema->leaf_column_id >= row_group_metadata.num_columns()) { + column_schema->leaf_column_id >= static_cast(row_group_metadata.columns.size())) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } - auto column_chunk = row_group_metadata.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_row_level_dictionary_filter(*column_schema, *column_chunk)) { + const auto& column_chunk = row_group_metadata.columns[column_schema->leaf_column_id]; + if (!column_chunk.__isset.meta_data || + !supports_row_level_dictionary_filter(*column_schema, column_chunk.meta_data)) { update_counter_if_not_null(_scan_profile.dict_filter_unsupported_columns, 1); continue; } @@ -1640,7 +1777,7 @@ void ParquetScanScheduler::prefetch_current_row_group_columns( const std::vector& scan_columns, bool* prefetched) { DORIS_CHECK(prefetched != nullptr); if (_current_merge_range_active || *prefetched || scan_columns.empty() || - _current_row_group_id < 0 || file_context.metadata == nullptr) { + _current_row_group_id < 0 || file_context.native_metadata == nullptr) { return; } *prefetched = true; @@ -1649,8 +1786,8 @@ void ParquetScanScheduler::prefetch_current_row_group_columns( // prefetch: callers decide which side to warm, and this helper only translates that selected // projection into physical column-chunk byte ranges for the current row group. file_context.prefetch_ranges( - build_row_group_prefetch_ranges(*file_context.metadata, file_schema, scan_columns, - _current_row_group_id), + build_row_group_prefetch_ranges(file_context.native_metadata->to_thrift(), file_schema, + scan_columns, _current_row_group_id), nullptr); } diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 9af54f261c683f..e855baa4dba454 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -60,6 +60,7 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +class NativeParquetMetadata; namespace detail { struct PredicateConjunctSchedule { @@ -121,6 +122,14 @@ Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr); +Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, + const ParquetScanRange& scan_range, bool enable_bloom_filter, + RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, + const RuntimeState* runtime_state = nullptr, + ParquetFileContext* file_context = nullptr); + IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -203,7 +212,7 @@ class ParquetScanScheduler { ParquetFileContext& file_context, const std::vector>& file_schema, const format::FileScanRequest& request, int row_group_idx, - const ::parquet::RowGroupMetaData& row_group_metadata); + const tparquet::RowGroup& row_group_metadata); void prefetch_current_row_group_columns( ParquetFileContext& file_context, diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index dd7287300569df..9c8019d6b6df8e 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -47,17 +47,26 @@ #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" +#include "format/parquet/parquet_block_split_bloom_filter.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" +#include "util/thrift_util.h" #include "util/unaligned.h" namespace doris::format::parquet { namespace { +bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, + const ParquetColumnSchema& column_schema, size_t page_idx, + ParquetColumnStatistics* page_statistics, + const cctz::time_zone* timezone); + enum class ParquetRowGroupPruneReason { NONE, // cannot prune; must read STATISTICS, // excluded by ZoneMap statistics @@ -65,6 +74,43 @@ enum class ParquetRowGroupPruneReason { BLOOM_FILTER, // excluded by bloom filter }; +Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, + const io::FileReaderSPtr& file, io::IOContext* io_ctx, + std::unique_ptr* result) { + if (result == nullptr || file == nullptr || !metadata.__isset.bloom_filter_offset) { + return Status::NotSupported("Parquet Bloom filter is unavailable"); + } + constexpr size_t MAX_BLOOM_HEADER_BYTES = 64; + const size_t header_read_size = + metadata.__isset.bloom_filter_length && metadata.bloom_filter_length > 0 + ? std::min(metadata.bloom_filter_length, MAX_BLOOM_HEADER_BYTES) + : MAX_BLOOM_HEADER_BYTES; + std::vector header_buffer(header_read_size); + size_t bytes_read = 0; + RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset, + Slice(header_buffer.data(), header_buffer.size()), &bytes_read, + io_ctx)); + tparquet::BloomFilterHeader header; + uint32_t header_size = cast_set(bytes_read); + RETURN_IF_ERROR(deserialize_thrift_msg(header_buffer.data(), &header_size, true, &header)); + if (!header.algorithm.__isset.BLOCK || !header.compression.__isset.UNCOMPRESSED || + !header.hash.__isset.XXHASH || header.numBytes <= 0) { + return Status::NotSupported("Unsupported Parquet Bloom filter encoding"); + } + + std::vector data(cast_set(header.numBytes)); + RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset + header_size, + Slice(data.data(), data.size()), &bytes_read, io_ctx)); + if (bytes_read != data.size()) { + return Status::Corruption("Truncated Parquet Bloom filter payload"); + } + auto bloom_filter = std::make_unique(); + RETURN_IF_ERROR(bloom_filter->init(reinterpret_cast(data.data()), data.size(), + segment_v2::HashStrategyPB::XX_HASH_64)); + *result = std::move(bloom_filter); + return Status::OK(); +} + bool bloom_logical_type_supported(const ParquetColumnSchema& column_schema) { if (column_schema.type == nullptr) { return false; @@ -821,6 +867,43 @@ ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( } } +ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone) { + ParquetColumnStatistics result; + if (statistics == nullptr || column_value_count < 0) { + return result; + } + + const bool has_null_count = statistics->__isset.null_count && statistics->null_count >= 0; + const int64_t null_count = has_null_count ? statistics->null_count : 0; + const bool has_not_null = has_null_count ? column_value_count > null_count : true; + const std::string* min_value = statistics->__isset.min_value + ? &statistics->min_value + : (statistics->__isset.min ? &statistics->min : nullptr); + const std::string* max_value = statistics->__isset.max_value + ? &statistics->max_value + : (statistics->__isset.max ? &statistics->max : nullptr); + + tparquet::ColumnIndex index; + index.__set_null_pages({!has_not_null}); + index.__set_null_counts({null_count}); + if (min_value != nullptr && max_value != nullptr) { + index.__set_min_values({*min_value}); + index.__set_max_values({*max_value}); + } + // Footer statistics and page indexes share the same little-endian physical encoding. Reusing + // one decoder keeps native row-group and page pruning identical for logical types and NaNs. + if (!build_native_page_statistics(index, column_schema, 0, &result, timezone)) { + return {}; + } + if (!has_null_count) { + result.has_null_count = false; + result.has_null = true; + } + return result; +} + bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, const VExprContextSPtrs& conjuncts, const ::parquet::BloomFilter& bloom_filter) { @@ -1097,6 +1180,303 @@ Status select_row_groups_by_metadata( namespace { +bool check_native_statistics(const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { + const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); + if (slot_indexes.empty()) { + return false; + } + ZoneMapEvalContext ctx; + for (const int slot_index : slot_indexes) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + std::shared_ptr zone_map; + if (chunk.__isset.meta_data) { + const auto& column_metadata = chunk.meta_data; + const auto* statistics = + column_metadata.__isset.statistics ? &column_metadata.statistics : nullptr; + zone_map = ParquetStatisticsUtils::MakeZoneMap( + ParquetStatisticsUtils::TransformColumnStatistics( + *column_schema, statistics, column_metadata.num_values, timezone)); + } + add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); + } + const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); + accumulate_zonemap_stats(ctx, pruning_stats); + return result == ZoneMapFilterResult::kNoMatch; +} + +bool is_native_dictionary_data_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN_DICTIONARY || + encoding == tparquet::Encoding::RLE_DICTIONARY; +} + +bool is_native_level_encoding(tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::RLE || encoding == tparquet::Encoding::BIT_PACKED; +} + +bool is_native_dictionary_encoded_chunk(const tparquet::ColumnMetaData& metadata) { + if (!metadata.__isset.dictionary_page_offset || metadata.dictionary_page_offset < 0) { + return false; + } + if (metadata.__isset.encoding_stats && !metadata.encoding_stats.empty()) { + bool has_dictionary_data_page = false; + for (const auto& encoding_stat : metadata.encoding_stats) { + if ((encoding_stat.page_type != tparquet::PageType::DATA_PAGE && + encoding_stat.page_type != tparquet::PageType::DATA_PAGE_V2) || + encoding_stat.count <= 0) { + continue; + } + if (!is_native_dictionary_data_encoding(encoding_stat.encoding)) { + return false; + } + has_dictionary_data_page = true; + } + return has_dictionary_data_page; + } + bool has_dictionary_encoding = false; + for (const auto encoding : metadata.encodings) { + if (is_native_dictionary_data_encoding(encoding)) { + has_dictionary_encoding = true; + } else if (!is_native_level_encoding(encoding)) { + return false; + } + } + return has_dictionary_encoding; +} + +const format::LocalColumnIndex* find_request_projection(const format::FileScanRequest& request, + format::LocalColumnId file_column_id) { + for (const auto& projection : request.predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; + } + } + for (const auto& projection : request.non_predicate_columns) { + if (projection.local_id() == file_column_id.value()) { + return &projection; + } + } + return nullptr; +} + +ParquetRowGroupPruneReason native_dictionary_prune_reason( + const tparquet::RowGroup& row_group, int row_group_idx, + const std::vector>& file_schema, + const format::FileScanRequest& request, const cctz::time_zone* timezone, + ParquetFileContext* file_context) { + if (file_context == nullptr || file_context->native_metadata == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_dictionary_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + const auto* projection = find_request_projection(request, *file_column_id); + if (column_schema == nullptr || projection == nullptr || column_schema->type == nullptr || + !column_schema->type_descriptor.is_string_like || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data || + (chunk.meta_data.type != tparquet::Type::BYTE_ARRAY && + chunk.meta_data.type != tparquet::Type::FIXED_LEN_BYTE_ARRAY) || + !is_native_dictionary_encoded_chunk(chunk.meta_data)) { + continue; + } + std::unique_ptr reader; + const std::vector ranges {{0, row_group.num_rows}}; + const std::unordered_map offset_indexes; + const auto status = NativeColumnReader::create( + *column_schema, projection, file_context->native_file, + file_context->native_metadata, row_group_idx, ranges, offset_indexes, timezone, + file_context->native_io_ctx, nullptr, file_context->native_page_cache_enabled, + file_context->native_page_cache_file_key, true, {}, &reader); + if (!status.ok() || reader == nullptr) { + continue; + } + auto dictionary_result = reader->dictionary_values(); + if (!dictionary_result.has_value()) { + continue; + } + auto dictionary = std::move(dictionary_result).value(); + std::vector values(dictionary->size()); + for (size_t value_idx = 0; value_idx < dictionary->size(); ++value_idx) { + dictionary->get(value_idx, values[value_idx]); + } + DictionaryEvalContext ctx; + ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { + .data_type = column_schema->type, + .values = std::move(values), + }); + if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == + ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::DICTIONARY; + } + } + return ParquetRowGroupPruneReason::NONE; +} + +ParquetRowGroupPruneReason native_bloom_filter_prune_reason( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, ParquetFileContext* file_context, + ParquetPruningStats* pruning_stats) { + if (file_context == nullptr || file_context->native_file == nullptr) { + return ParquetRowGroupPruneReason::NONE; + } + const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( + request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); + for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + const auto file_column_id = file_column_id_by_block_position(request, slot_index); + if (!file_column_id.has_value()) { + continue; + } + const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); + if (column_schema == nullptr || column_schema->type == nullptr || + !bloom_filter_supported(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[column_schema->leaf_column_id]; + if (!chunk.__isset.meta_data) { + continue; + } + std::unique_ptr bloom_filter; + Status status; + { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->bloom_filter_read_time); + status = read_native_bloom_filter(chunk.meta_data, file_context->native_file, + file_context->native_io_ctx, &bloom_filter); + } + if (!status.ok() || bloom_filter == nullptr) { + continue; + } + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema->type, + .bloom_filter = bloom_filter.get(), + }); + if (VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch) { + return ParquetRowGroupPruneReason::BLOOM_FILTER; + } + } + return ParquetRowGroupPruneReason::NONE; +} + +int64_t native_requested_compressed_bytes( + const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request) { + std::set leaf_column_ids; + auto collect_projection = [&](const format::LocalColumnIndex& projection) { + const int32_t local_id = projection.local_id(); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return; + } + collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); + }; + for (const auto& projection : request.predicate_columns) { + collect_projection(projection); + } + for (const auto& projection : request.non_predicate_columns) { + collect_projection(projection); + } + int64_t bytes = 0; + for (const int leaf_column_id : leaf_column_ids) { + if (leaf_column_id < 0 || leaf_column_id >= static_cast(row_group.columns.size())) { + continue; + } + const auto& chunk = row_group.columns[leaf_column_id]; + if (chunk.__isset.meta_data && chunk.meta_data.total_compressed_size > 0) { + bytes += chunk.meta_data.total_compressed_size; + } + } + return bytes; +} + +} // namespace + +Status select_row_groups_by_metadata( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, + const RuntimeState* runtime_state, ParquetFileContext* file_context) { + int64_t timer_sink = 0; + SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink + : &pruning_stats->row_group_filter_time); + if (selected_row_groups == nullptr) { + return Status::InvalidArgument("selected_row_groups is null"); + } + selected_row_groups->clear(); + const size_t candidate_size = candidate_row_groups == nullptr ? metadata.row_groups.size() + : candidate_row_groups->size(); + if (pruning_stats != nullptr) { + pruning_stats->total_row_groups = cast_set(candidate_size); + } + selected_row_groups->reserve(candidate_size); + for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { + const int row_group_idx = candidate_row_groups == nullptr + ? static_cast(candidate_idx) + : (*candidate_row_groups)[candidate_idx]; + DORIS_CHECK(row_group_idx >= 0 && + row_group_idx < static_cast(metadata.row_groups.size())); + const auto& row_group = metadata.row_groups[row_group_idx]; + ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; + if (has_expr_zonemap_filter(request, runtime_state) && + check_native_statistics(row_group, file_schema, request, pruning_stats, timezone)) { + prune_reason = ParquetRowGroupPruneReason::STATISTICS; + } + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + prune_reason = native_dictionary_prune_reason(row_group, row_group_idx, file_schema, + request, timezone, file_context); + } + if (prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { + prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, + file_context, pruning_stats); + } + if (prune_reason == ParquetRowGroupPruneReason::NONE) { + selected_row_groups->push_back(row_group_idx); + continue; + } + if (pruning_stats != nullptr) { + pruning_stats->filtered_group_rows += row_group.num_rows; + pruning_stats->filtered_bytes += + native_requested_compressed_bytes(row_group, file_schema, request); + if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { + ++pruning_stats->filtered_row_groups_by_statistics; + } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { + ++pruning_stats->filtered_row_groups_by_dictionary; + } else { + ++pruning_stats->filtered_row_groups_by_bloom_filter; + } + } + } + return Status::OK(); +} + +namespace { + template bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, const ParquetColumnSchema& column_schema, size_t page_idx, @@ -1138,7 +1518,7 @@ bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& colu const ParquetColumnSchema& column_schema, size_t page_idx, ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - switch (column_schema.descriptor->physical_type()) { + switch (column_schema.type_descriptor.physical_type) { case ::parquet::Type::BYTE_ARRAY: { const auto typed_index = std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); @@ -1201,7 +1581,7 @@ bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_inde const ParquetColumnSchema& column_schema, size_t page_idx, ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { DORIS_CHECK(column_schema.type != nullptr); - switch (column_schema.descriptor->physical_type()) { + switch (column_schema.type_descriptor.physical_type) { case ::parquet::Type::BOOLEAN: return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, page_idx, DecodedValueKind::BOOL, @@ -1674,13 +2054,19 @@ bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, page_idx >= column_index.null_counts.size()) { return false; } + const int64_t null_count = column_index.null_counts[page_idx]; + if (null_count < 0 || (column_index.null_pages[page_idx] && null_count == 0)) { + // Contradictory optional index metadata must disable pruning; treating it as an empty + // null set can make IS NULL/IS NOT NULL discard rows without reading the data page. + return false; + } page_statistics->has_null_count = true; - page_statistics->has_null = column_index.null_counts[page_idx] > 0; + page_statistics->has_null = null_count > 0; page_statistics->has_not_null = !column_index.null_pages[page_idx]; if (!page_statistics->has_not_null) { return true; } - switch (column_schema.descriptor->physical_type()) { + switch (column_schema.type_descriptor.physical_type) { case ::parquet::Type::BOOLEAN: return set_native_page_scalar_min_max(column_index, column_schema, page_idx, DecodedValueKind::BOOL, page_statistics, @@ -1709,12 +2095,12 @@ bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, } const auto& min_value = column_index.min_values[page_idx]; const auto& max_value = column_index.max_values[page_idx]; - const bool fixed = - column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + const bool fixed = column_schema.type_descriptor.physical_type == + ::parquet::Type::FIXED_LEN_BYTE_ARRAY; if (fixed && - (column_schema.descriptor->type_length() <= 0 || - min_value.size() != static_cast(column_schema.descriptor->type_length()) || - max_value.size() != static_cast(column_schema.descriptor->type_length()))) { + (column_schema.type_descriptor.fixed_length <= 0 || + min_value.size() != static_cast(column_schema.type_descriptor.fixed_length) || + max_value.size() != static_cast(column_schema.type_descriptor.fixed_length))) { return false; } const auto kind = fixed ? DecodedValueKind::FIXED_BINARY : DecodedValueKind::BINARY; @@ -1784,7 +2170,7 @@ Status select_row_group_ranges_by_native_page_index( continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->descriptor == nullptr) { + if (column_schema == nullptr) { continue; } const auto index_it = page_indexes.find(column_schema->leaf_column_id); @@ -1836,8 +2222,7 @@ Status select_row_group_ranges_by_native_page_index( collect_request_leaf_schemas(file_schema, request, &leaves); for (const auto* leaf : leaves) { const auto index_it = page_indexes.find(leaf->leaf_column_id); - if (index_it == page_indexes.end() || leaf->descriptor == nullptr || - leaf->descriptor->max_repetition_level() != 0) { + if (index_it == page_indexes.end() || leaf->max_repetition_level != 0) { continue; } const auto& offset_index = index_it->second.offset_index; diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 737fc4b58ae372..184a51997696e9 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -54,6 +54,7 @@ struct ZoneMap; namespace doris::format::parquet { struct ParquetColumnSchema; +struct ParquetFileContext; // ============================================================================ // ============================================================================ @@ -145,6 +146,10 @@ struct ParquetStatisticsUtils { const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone = nullptr); + static ParquetColumnStatistics TransformColumnStatistics( + const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, + int64_t column_value_count, const cctz::time_zone* timezone = nullptr); + static bool TransformColumnIndexStatistics( const std::shared_ptr<::parquet::ColumnIndex>& column_index, const ParquetColumnSchema& column_schema, size_t page_idx, @@ -163,6 +168,14 @@ Status select_row_groups_by_metadata( ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr); +Status select_row_groups_by_metadata( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector* candidate_row_groups, + std::vector* selected_row_groups, bool enable_bloom_filter, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, + const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr); + Status select_row_group_ranges_by_page_index( ::parquet::ParquetFileReader* file_reader, const std::vector>& file_schema, diff --git a/be/src/format_v2/parquet/reader/count_column_reader.cpp b/be/src/format_v2/parquet/reader/count_column_reader.cpp index a3f316c5e8558f..171107107f1a38 100644 --- a/be/src/format_v2/parquet/reader/count_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/count_column_reader.cpp @@ -22,7 +22,7 @@ #include #include "common/config.h" -#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/reader/native/level_reader.h" @@ -73,7 +73,7 @@ Status find_count_leaf(const ParquetColumnSchema& schema, return Status::InternalError("Unknown Parquet schema kind for column {}", schema.name); } -FieldSchema* find_physical_leaf(FieldSchema* field, int physical_column_index) { +NativeFieldSchema* find_physical_leaf(NativeFieldSchema* field, int physical_column_index) { DORIS_CHECK(field != nullptr); if (field->children.empty()) { return field->physical_column_index == physical_column_index ? field : nullptr; @@ -123,7 +123,7 @@ Status CountColumnReader::create(io::FileReaderSPtr file, const NativeParquetMet DORIS_CHECK(root_schema.local_id >= 0); auto* root_field = - const_cast(metadata->schema().get_column(root_schema.local_id)); + const_cast(metadata->schema().get_column(root_schema.local_id)); DORIS_CHECK(root_field != nullptr); auto* leaf_field = find_physical_leaf(root_field, leaf_schema->leaf_column_id); if (leaf_field == nullptr) { diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 7f6dd9254757f4..a696af8ffddd36 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -17,6 +17,7 @@ #include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include #include #include #include @@ -30,9 +31,11 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "core/column/column.h" +#include "core/column/column_vector.h" #include "core/custom_allocator.h" #include "core/data_type_serde/data_type_serde.h" -#include "format/parquet/schema_desc.h" +#include "core/data_type_serde/parquet_timestamp.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" @@ -41,6 +44,7 @@ #include "storage/cache/page_cache.h" #include "util/bit_util.h" #include "util/block_compression.h" +#include "util/unaligned.h" namespace cctz { class time_zone; @@ -54,6 +58,11 @@ struct IOContext; namespace doris::format::parquet::native { +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available) { + return session_cache_enabled && !storage_cache_disabled && cache_available && header_available; +} + ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { if (created_by.empty()) { return {}; @@ -69,8 +78,10 @@ Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size bool parquet_816_padding, ColumnChunkRange* range) { DORIS_CHECK(range != nullptr); int64_t start = metadata.data_page_offset; - if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset >= 0 && + if (metadata.__isset.dictionary_page_offset && metadata.dictionary_page_offset > 0 && metadata.dictionary_page_offset < start) { + // Some writers use dictionary_page_offset=0 as an absence sentinel. Range validation must + // follow has_dict_page() or the native reader starts at the Parquet magic bytes. start = metadata.dictionary_page_offset; } const int64_t length = metadata.total_compressed_size; @@ -129,6 +140,128 @@ bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunk namespace { +Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, + const ParquetInt96Timestamp& value, + const cctz::time_zone& timezone) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_PER_DAY = 86400000000LL; + static constexpr int64_t MICROS_PER_SECOND = 1000000LL; + + // Arrow normalized out-of-day INT96 nanos before the V2 native path replaced it. Preserve that + // file-compatibility invariant here; rejecting the raw nanos loses otherwise valid year-0 and + // pre-epoch timestamps used by existing Parquet files. + const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + // The compatibility cast truncates the signed nanos field itself. Normalizing it to a positive + // time-of-day first changes negative out-of-day values by one microsecond. + const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; + if (timestamp_micros < std::numeric_limits::min() || + timestamp_micros > std::numeric_limits::max()) { + return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); + } + + const int64_t micros = static_cast(timestamp_micros); + int64_t epoch_seconds = micros / MICROS_PER_SECOND; + int64_t micros_of_second = micros % MICROS_PER_SECOND; + if (micros_of_second < 0) { + micros_of_second += MICROS_PER_SECOND; + --epoch_seconds; + } + DateV2Value datetime; + datetime.from_unixtime(epoch_seconds, timezone); + datetime.set_microsecond(static_cast(micros_of_second)); + if (!datetime.is_valid_date()) { + return Status::DataQualityError("Parquet INT96 timestamp is outside the Doris range"); + } + data.push_back(datetime); + return Status::OK(); +} + +class V2Int96DateTimeConsumer final : public ParquetFixedValueConsumer { +public: + V2Int96DateTimeConsumer(IColumn& column, const ParquetDecodeContext& context, + ParquetMaterializationState* state) + : _data(assert_cast(column).get_data()), _state(state) { + static const auto utc = cctz::utc_time_zone(); + _timezone = context.timezone == nullptr ? &utc : context.timezone; + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + DORIS_CHECK_EQ(value_width, sizeof(ParquetInt96Timestamp)); + const size_t old_size = _data.size(); + for (size_t row = 0; row < num_values; ++row) { + const auto value = unaligned_load( + values + row * sizeof(ParquetInt96Timestamp)); + const auto status = append_v2_int96_datetime(_data, value, *_timezone); + if (!status.ok()) { + if (_state != nullptr && _state->mark_conversion_failure(_data.size())) { + _data.emplace_back(); + continue; + } + _data.resize(old_size); + return status; + } + } + return Status::OK(); + } + +private: + ColumnDateTimeV2::Container& _data; + ParquetMaterializationState* _state; + const cctz::time_zone* _timezone = nullptr; +}; + +class RejectV2Int96BinaryConsumer final : public ParquetBinaryValueConsumer { +public: + Status consume(const StringRef*, size_t) override { + return Status::NotSupported("INT96 cannot be decoded from binary Parquet values"); + } +}; + +Status read_v2_int96_datetime(IColumn& column, ParquetDecodeSource& source, + const ParquetDecodeContext& context, size_t num_values, + ParquetMaterializationState& state) { + V2Int96DateTimeConsumer consumer(column, context, &state); + if (context.encoding != ParquetValueEncoding::DICTIONARY) { + return source.decode_fixed_values(num_values, consumer); + } + if (state.dictionary_generation != source.dictionary_generation()) { + state.typed_dictionary = column.clone_empty(); + auto* output_null_map = state.begin_dictionary_conversion(source.dictionary_size()); + V2Int96DateTimeConsumer dictionary_consumer(*state.typed_dictionary, context, &state); + RejectV2Int96BinaryConsumer binary_consumer; + const auto dictionary_status = + source.decode_dictionary(dictionary_consumer, binary_consumer); + state.end_dictionary_conversion(output_null_map); + RETURN_IF_ERROR(dictionary_status); + DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); + state.dictionary_generation = source.dictionary_generation(); + } + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); + DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); + const size_t old_size = column.size(); + column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), + state.dictionary_indices.data() + num_values); + if (state.can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < num_values; ++row) { + if (!state.dictionary_conversion_failures.empty() && + state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { + state.mark_conversion_failure(old_size + row); + } + } + } + return Status::OK(); +} + +Status read_native_or_serde(IColumn& column, const DataTypeSerDe& serde, + ParquetDecodeSource& source, const ParquetDecodeContext& context, + size_t num_values, ParquetMaterializationState& state) { + if (context.physical_type == ParquetPhysicalType::INT96 && + check_and_get_column(&column) != nullptr) { + return read_v2_int96_datetime(column, source, context, num_values, state); + } + return serde.read_column_from_parquet(column, source, context, num_values, state); +} + Status translate_value_encoding(tparquet::Encoding::type encoding, ParquetValueEncoding* translated) { DORIS_CHECK(translated != nullptr); @@ -175,7 +308,7 @@ Status decode_selected_values(IColumn& column, const DataTypeSerDe& serde, Decod switch (read_type) { case ColumnSelectVector::CONTENT: RETURN_IF_ERROR( - serde.read_column_from_parquet(column, decoder, context, run_length, state)); + read_native_or_serde(column, serde, decoder, context, run_length, state)); break; case ColumnSelectVector::NULL_DATA: column.insert_many_defaults(run_length); @@ -260,8 +393,8 @@ Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& ser SCOPED_RAW_TIMER(materialization_time); SelectedDecodeSource selected_source(decoder, selection); - return serde.read_column_from_parquet(column, selected_source, context, - selection.selected_values, state); + return read_native_or_serde(column, serde, selected_source, context, selection.selected_values, + state); } } // namespace @@ -269,8 +402,8 @@ Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& ser template ColumnChunkReader::ColumnChunkReader( io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, - FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_rows, - io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + size_t total_rows, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, const ColumnChunkRange* chunk_range) : _field_schema(field_schema), _max_rep_level(field_schema->repetition_level), @@ -549,6 +682,9 @@ Status ColumnChunkReader::load_page_data() { } if (!page_loaded) { + const bool prepare_cache_payload = can_prepare_page_cache_payload( + _page_read_ctx.enable_parquet_file_page_cache, config::disable_storage_page_cache, + StoragePageCache::instance() != nullptr, !_page_reader->header_bytes().empty()); if (_block_compress_codec != nullptr) { Slice compressed_data; RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); @@ -563,7 +699,7 @@ Status ColumnChunkReader::load_page_data() { size_t rl = header_v2.repetition_levels_byte_length; size_t dl = header_v2.definition_levels_byte_length; size_t level_sz = rl + dl; - if (level_sz > 0) { + if (prepare_cache_payload && level_sz > 0) { level_bytes.resize(level_sz); memcpy(level_bytes.data(), compressed_data.data, level_sz); } @@ -591,10 +727,7 @@ Status ColumnChunkReader::load_page_data() { bool cache_payload_decompressed = should_cache_decompressed( header, _metadata, _page_read_ctx.data_page_v2_always_compressed); - if (_page_read_ctx.enable_parquet_file_page_cache && - !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && - !_page_reader->header_bytes().empty()) { + if (prepare_cache_payload) { if (cache_payload_decompressed) { _insert_page_into_cache(level_bytes, _page_data); _chunk_statistics.page_cache_decompressed_write_counter += 1; @@ -610,9 +743,7 @@ Status ColumnChunkReader::load_page_data() { } else { // no compression on this page, use the data directly _page_data = Slice(compressed_data.data, compressed_data.size); - if (_page_read_ctx.enable_parquet_file_page_cache && - !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr) { + if (prepare_cache_payload) { _insert_page_into_cache(level_bytes, _page_data); _chunk_statistics.page_cache_decompressed_write_counter += 1; } @@ -627,7 +758,7 @@ Status ColumnChunkReader::load_page_data() { size_t rl = header_v2.repetition_levels_byte_length; size_t dl = header_v2.definition_levels_byte_length; size_t level_sz = rl + dl; - if (level_sz > 0) { + if (prepare_cache_payload && level_sz > 0) { level_bytes.resize(level_sz); memcpy(level_bytes.data(), uncompressed_data.data, level_sz); } @@ -636,8 +767,7 @@ Status ColumnChunkReader::load_page_data() { // copy page data out _page_data = Slice(uncompressed_data.data, uncompressed_data.size); // Optionally cache uncompressed data for uncompressed pages - if (_page_read_ctx.enable_parquet_file_page_cache && - !config::disable_storage_page_cache && StoragePageCache::instance() != nullptr) { + if (prepare_cache_payload) { _insert_page_into_cache(level_bytes, _page_data); _chunk_statistics.page_cache_decompressed_write_counter += 1; } diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 9b7c1f27571ab8..8106070bae9050 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -31,6 +31,7 @@ #include "core/data_type/data_type.h" #include "core/data_type_serde/parquet_decode_source.h" #include "format/parquet/parquet_common.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" @@ -39,8 +40,6 @@ namespace doris { class BlockCompressionCodec; class DataTypeSerDe; -struct FieldSchema; - namespace io { class BufferedStreamReader; struct IOContext; @@ -49,7 +48,6 @@ struct IOContext; } // namespace doris namespace doris::format::parquet::native { -using ::doris::FieldSchema; using ::doris::ColumnString; struct ColumnChunkRange { @@ -67,6 +65,8 @@ Status compute_column_chunk_range(const tparquet::ColumnMetaData& metadata, size bool parquet_816_padding, ColumnChunkRange* range); bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunkRange& chunk_range, int64_t data_page_offset, int64_t row_count); +bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, + bool cache_available, bool header_available); struct ColumnChunkReaderStatistics { int64_t decompress_time = 0; @@ -99,7 +99,7 @@ struct ColumnChunkReaderStatistics { * // Create chunk reader * ColumnChunkReader chunk_reader(BufferedStreamReader* reader, * tparquet::ColumnChunk* column_chunk, - * FieldSchema* fieldSchema); + * NativeFieldSchema* fieldSchema); * // Initialize chunk reader * chunk_reader.init(); * while (chunk_reader.has_next_page()) { @@ -117,7 +117,7 @@ template class ColumnChunkReader { public: ColumnChunkReader(io::BufferedStreamReader* reader, tparquet::ColumnChunk* column_chunk, - FieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, + NativeFieldSchema* field_schema, const tparquet::OffsetIndex* offset_index, size_t total_row, io::IOContext* io_ctx, const ParquetPageReadContext& page_read_ctx, const ColumnChunkRange* chunk_range = nullptr); @@ -299,7 +299,7 @@ class ColumnChunkReader { } ColumnChunkReaderState _state = NOT_INIT; - FieldSchema* _field_schema = nullptr; + NativeFieldSchema* _field_schema = nullptr; const level_t _max_rep_level; const level_t _max_def_level; diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index fd783fd32a0b34..25b7ec3795734a 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -37,7 +37,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" -#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "io/fs/tracing_file_reader.h" @@ -103,6 +103,53 @@ bool is_direct_binary_type(PrimitiveType type) { return is_string_type(type) || type == TYPE_VARBINARY; } +IColumn::Filter* conversion_failure_map(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn::Filter* output_null_map, + IColumn::Filter* compatibility_scratch) { + const auto& schema = field.parquet_schema; + const bool is_utc_timestamp = + field.physical_type == tparquet::Type::INT96 || + (field.physical_type == tparquet::Type::INT64 && + ((schema.__isset.logicalType && schema.logicalType.__isset.TIMESTAMP && + schema.logicalType.TIMESTAMP.isAdjustedToUTC) || + (schema.__isset.converted_type && + (schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MILLIS || + schema.converted_type == tparquet::ConvertedType::TIMESTAMP_MICROS)))); + if (!strict_mode && output_null_map != nullptr && is_utc_timestamp && + remove_nullable(target_type)->get_primitive_type() == TYPE_DATETIMEV2) { + // Legacy UTC timestamp conversion kept out-of-range values as DATETIME defaults. Local + // timestamps intentionally keep non-strict NULL-on-overflow behavior because timezone + // conversion is not part of their representation. + compatibility_scratch->resize_fill(output_null_map->size(), 0); + return compatibility_scratch; + } + return output_null_map; +} + +void mark_local_timestamp_defaults(const NativeFieldSchema& field, const DataTypePtr& target_type, + bool strict_mode, IColumn& data_column, + IColumn::Filter* output_null_map, size_t start_row) { + const auto& schema = field.parquet_schema; + const bool is_local_timestamp = + field.physical_type == tparquet::Type::INT64 && schema.__isset.logicalType && + schema.logicalType.__isset.TIMESTAMP && !schema.logicalType.TIMESTAMP.isAdjustedToUTC; + if (strict_mode || output_null_map == nullptr || !is_local_timestamp || + remove_nullable(target_type)->get_primitive_type() != TYPE_DATETIMEV2) { + return; + } + auto& values = assert_cast(data_column).get_data(); + DORIS_CHECK_EQ(values.size(), output_null_map->size()); + for (size_t row = start_row; row < values.size(); ++row) { + if ((*output_null_map)[row] == 0 && !values[row].is_valid_date()) { + // Local timestamps before Doris' representable calendar can materialize as a zero + // date without a SerDe error. Preserve non-strict scan semantics by nulling only that + // sentinel; physical NULLs and valid local timestamps keep their original map bits. + (*output_null_map)[row] = 1; + } + } +} + // The target SerDe can fuse physical decode with these logical type changes. Less common schema // changes retain the generic file-format converter as a compatibility path: the decoder still // exposes raw spans, but the source SerDe first materializes a reusable source column before the @@ -115,12 +162,12 @@ bool serde_can_materialize_directly(const DataTypePtr& source_type, (source == TYPE_FLOAT && target == TYPE_DOUBLE) || (is_direct_decimal_type(source) && is_direct_decimal_type(target)) || // Parquet STRING and VARBINARY share BYTE_ARRAY bytes. Materializing through the target - // SerDe preserves those bytes and avoids a converter whose scratch column uses the v1 + // SerDe preserves those bytes and avoids a converter whose scratch column uses the // String representation instead of the native ColumnVarbinary representation. (is_direct_binary_type(source) && is_direct_binary_type(target)); } -Status init_decode_context(const FieldSchema& field, const cctz::time_zone* ctz, +Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, ParquetDecodeContext* context) { DORIS_CHECK(context != nullptr); switch (field.physical_type) { @@ -269,13 +316,31 @@ Status init_decode_context(const FieldSchema& field, const cctz::time_zone* ctz, } // namespace #ifdef BE_TEST -Status init_decode_context_for_test(const FieldSchema& field, const cctz::time_zone* ctz, +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, ParquetDecodeContext* context) { return init_decode_context(field, ctz, context); } + +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode) { + IColumn::Filter output_null_map; + output_null_map.resize_fill(1, 0); + IColumn::Filter compatibility_scratch; + return conversion_failure_map(field, target_type, strict_mode, &output_null_map, + &compatibility_scratch) == &compatibility_scratch; +} + +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row) { + mark_local_timestamp_defaults(field, target_type, strict_mode, data_column, output_null_map, + start_row); +} #endif -static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, +static void fill_struct_null_map(NativeFieldSchema* field, NullMap& null_map, const std::vector& rep_levels, const std::vector& def_levels) { size_t num_levels = def_levels.size(); @@ -298,7 +363,7 @@ static void fill_struct_null_map(FieldSchema* field, NullMap& null_map, null_map.resize(pos); } -static Status fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offsets_data, +static Status fill_array_offset(NativeFieldSchema* field, ColumnArray::Offsets64& offsets_data, NullMap* null_map_ptr, const std::vector& rep_levels, const std::vector& def_levels) { size_t num_levels = rep_levels.size(); @@ -348,7 +413,7 @@ static Status fill_array_offset(FieldSchema* field, ColumnArray::Offsets64& offs return Status::OK(); } -Status ColumnReader::create(io::FileReaderSPtr file, FieldSchema* field, +Status ColumnReader::create(io::FileReaderSPtr file, NativeFieldSchema* field, const tparquet::RowGroup& row_group, const RowRanges& row_ranges, const cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, size_t max_buf_size, @@ -503,7 +568,7 @@ void ColumnReader::_generate_read_ranges(RowRange page_row_range, RowRanges* res template Status ScalarColumnReader::init( - io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, RuntimeState* state, + io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state, const std::string& page_cache_file_key, const ParquetReaderCompat& compat, bool enable_strict_mode) { _field_schema = field; @@ -732,9 +797,20 @@ Status ScalarColumnReader::_read_values(size_t num_ DORIS_CHECK(_serde != nullptr); // Keep selected-row cardinality stable: non-strict conversion failures append a nested // default and mark this matching nullable output row instead of shortening the column. - _materialization_state.conversion_failure_null_map = map_data_column; - return _chunk_reader->materialize_values(data_column, *_serde, _decode_context, - _materialization_state, _select_vector); + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = + conversion_failure_map(*_field_schema, type, _materialization_state.enable_strict_mode, + map_data_column, &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values(data_column, *_serde, _decode_context, + _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + return status; } /** @@ -796,9 +872,20 @@ Status ScalarColumnReader::_read_nested_column( DORIS_CHECK(_serde != nullptr); // Nested materialization must preserve the same value/null-map row alignment invariant. - _materialization_state.conversion_failure_null_map = map_data_column; - RETURN_IF_ERROR(_chunk_reader->materialize_values(data_column, *_serde, _decode_context, - _materialization_state, _select_vector)); + IColumn::Filter compatibility_scratch; + _materialization_state.conversion_failure_null_map = conversion_failure_map( + *_field_schema, type, _materialization_state.enable_strict_mode, map_data_column, + &compatibility_scratch); + const size_t materialization_start_row = data_column->size(); + const auto status = _chunk_reader->materialize_values( + data_column, *_serde, _decode_context, _materialization_state, _select_vector); + _materialization_state.conversion_failure_null_map = nullptr; + if (status.ok()) { + mark_local_timestamp_defaults(*_field_schema, type, + _materialization_state.enable_strict_mode, *data_column, + map_data_column, materialization_start_row); + } + RETURN_IF_ERROR(status); if (!_ancestor_null_indices.empty()) { RETURN_IF_ERROR(_chunk_reader->skip_values(_ancestor_null_indices.size(), false)); } @@ -1000,7 +1087,7 @@ Status ScalarColumnReader::_try_load_dict_page(bool template Status ScalarColumnReader::read_column_data( ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, FilterMap& filter_map, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size) { const DataTypePtr target_type = remove_nullable(type); @@ -1148,17 +1235,18 @@ Status ScalarColumnReader::read_column_data( return finish_logical_conversion(); } -Status ArrayColumnReader::init(std::unique_ptr element_reader, FieldSchema* field) { +Status ArrayColumnReader::init(std::unique_ptr element_reader, + NativeFieldSchema* field) { _field_schema = field; _element_reader = std::move(element_reader); return Status::OK(); } -Status ArrayColumnReader::read_column_data( - ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, FilterMap& filter_map, - size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, - int64_t real_column_size) { +Status ArrayColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { MutableColumnPtr data_column; NullMap* null_map_ptr = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); @@ -1184,8 +1272,8 @@ Status ArrayColumnReader::read_column_data( (assert_cast(remove_nullable(type).get()))->get_nested_type(); // read nested column RETURN_IF_ERROR(_element_reader->read_column_data(element_column, element_type, - root_node->get_element_node(), filter_map, - batch_size, read_rows, eof, is_dict_filter)); + root_node->element(), filter_map, batch_size, + read_rows, eof, is_dict_filter)); if (*read_rows == 0) { return Status::OK(); } @@ -1205,18 +1293,18 @@ Status ArrayColumnReader::read_column_data( } Status MapColumnReader::init(std::unique_ptr key_reader, - std::unique_ptr value_reader, FieldSchema* field) { + std::unique_ptr value_reader, NativeFieldSchema* field) { _field_schema = field; _key_reader = std::move(key_reader); _value_reader = std::move(value_reader); return Status::OK(); } -Status MapColumnReader::read_column_data( - ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, FilterMap& filter_map, - size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, - int64_t real_column_size) { +Status MapColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { MutableColumnPtr data_column; NullMap* null_map_ptr = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); @@ -1251,16 +1339,15 @@ Status MapColumnReader::read_column_data( bool value_eof = false; int64_t orig_col_column_size = key_column->size(); - RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->get_key_node(), + RETURN_IF_ERROR(_key_reader->read_column_data(key_column, key_type, root_node->key(), filter_map, batch_size, &key_rows, &key_eof, is_dict_filter)); while (value_rows < key_rows && !value_eof) { size_t loop_rows = 0; RETURN_IF_ERROR(_value_reader->read_column_data( - value_column, value_type, root_node->get_value_node(), filter_map, - key_rows - value_rows, &loop_rows, &value_eof, is_dict_filter, - key_column->size() - orig_col_column_size)); + value_column, value_type, root_node->value(), filter_map, key_rows - value_rows, + &loop_rows, &value_eof, is_dict_filter, key_column->size() - orig_col_column_size)); value_rows += loop_rows; } if (UNLIKELY(key_rows != value_rows)) { @@ -1275,17 +1362,12 @@ Status MapColumnReader::read_column_data( return Status::OK(); } - const size_t key_values = key_column->size() - orig_col_column_size; if (UNLIKELY(key_column->size() != value_column->size())) { return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", key_column->size(), value_column->size()); } - if (const auto* nullable_keys = typeid_cast(key_column.get()); UNLIKELY( - nullable_keys != nullptr && - nullable_keys->has_null(orig_col_column_size, orig_col_column_size + key_values))) { - // Doris MAP keys are non-null even if a malformed/evolved file exposes a nullable child. - return Status::Corruption("Parquet map contains a null key"); - } + // Non-strict conversion can persist nullable Doris MAP keys, so the native reader must + // preserve writer output instead of reclassifying an otherwise valid file as corrupt. const auto& key_rep_levels = _key_reader->get_rep_level(); // fill offset and null map // The key leaf is the canonical outer MAP shape. A nested value has additional repetition @@ -1309,7 +1391,7 @@ Status MapColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_s Status StructColumnReader::init( std::unordered_map>&& child_readers, - FieldSchema* field) { + NativeFieldSchema* field) { _field_schema = field; _child_readers = std::move(child_readers); return Status::OK(); @@ -1330,11 +1412,11 @@ Status StructColumnReader::read_column_levels(FilterMap& filter_map, size_t batc return Status::InternalError("Struct {} has no physical reader for levels", _field_schema->name); } -Status StructColumnReader::read_column_data( - ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, FilterMap& filter_map, - size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, - int64_t real_column_size) { +Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, + const std::shared_ptr& root_node, + FilterMap& filter_map, size_t batch_size, + size_t* read_rows, bool* eof, bool is_dict_filter, + int64_t real_column_size) { MutableColumnPtr data_column; NullMap* null_map_ptr = nullptr; doris_column = IColumn::mutate(std::move(doris_column)); @@ -1384,13 +1466,13 @@ Status StructColumnReader::read_column_data( ColumnPtr& doris_field = doris_struct.get_column_ptr(i); auto& doris_type = doris_struct_type->get_element(i); auto& doris_name = doris_struct_type->get_element_name(i); - if (!root_node->children_column_exists(doris_name)) { + if (!root_node->has_child(doris_name)) { missing_column_idxs.push_back(i); VLOG_DEBUG << "[ParquetReader] Missing column in schema: column_idx[" << i << "], doris_name: " << doris_name << " (column not exists in root node)"; continue; } - auto file_name = root_node->children_file_column_name(doris_name); + auto file_name = root_node->file_child_name(doris_name); // Check if this is a SkipReadingReader - we should skip it when choosing reference column // because SkipReadingReader doesn't know the actual data size in nested context @@ -1413,8 +1495,8 @@ Status StructColumnReader::read_column_data( not_missing_column_id = i; not_missing_orig_column_size = doris_field->size(); RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( - doris_field, doris_type, root_node->get_children_node(doris_name), filter_map, - batch_size, &field_rows, &field_eof, is_dict_filter)); + doris_field, doris_type, root_node->child(doris_name), filter_map, batch_size, + &field_rows, &field_eof, is_dict_filter)); *read_rows = field_rows; *eof = field_eof; reference_parent_shape = parent_shape(*_child_readers[file_name]); @@ -1431,9 +1513,8 @@ Status StructColumnReader::read_column_data( while (field_rows < *read_rows && !field_eof) { size_t loop_rows = 0; RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( - doris_field, doris_type, root_node->get_children_node(doris_name), - filter_map, *read_rows - field_rows, &loop_rows, &field_eof, - is_dict_filter)); + doris_field, doris_type, root_node->child(doris_name), filter_map, + *read_rows - field_rows, &loop_rows, &field_eof, is_dict_filter)); field_rows += loop_rows; } if (UNLIKELY(*read_rows != field_rows)) { @@ -1518,12 +1599,12 @@ Status StructColumnReader::read_column_data( auto& doris_field = doris_struct.get_column_ptr(idx); auto& doris_type = const_cast(doris_struct_type->get_element(idx)); auto& doris_name = const_cast(doris_struct_type->get_element_name(idx)); - auto file_name = root_node->children_file_column_name(doris_name); + auto file_name = root_node->file_child_name(doris_name); size_t field_rows = 0; bool field_eof = false; RETURN_IF_ERROR(_child_readers[file_name]->read_column_data( - doris_field, doris_type, root_node->get_children_node(doris_name), filter_map, + doris_field, doris_type, root_node->child(doris_name), filter_map, missing_column_sz, &field_rows, &field_eof, is_dict_filter, missing_column_sz)); } diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 2b70a416f07c88..f088d4e7d44d98 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -32,7 +32,7 @@ #include "format/column_type_convert.h" #include "format/generic_reader.h" #include "format/parquet/parquet_common.h" -#include "format/table/table_schema_change_helper.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader_writer_fwd.h" @@ -46,12 +46,18 @@ struct IOContext; } // namespace doris::io namespace doris::format::parquet::native { -using ::doris::FieldSchema; using ::doris::ColumnString; #ifdef BE_TEST -Status init_decode_context_for_test(const FieldSchema& field, const cctz::time_zone* ctz, +Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, ParquetDecodeContext* context); +bool preserves_timestamp_conversion_default_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, + bool strict_mode); +void mark_local_timestamp_defaults_for_test(const NativeFieldSchema& field, + const DataTypePtr& target_type, bool strict_mode, + IColumn& data_column, IColumn::Filter* output_null_map, + size_t start_row); #endif class ColumnReader { @@ -178,7 +184,7 @@ class ColumnReader { : _row_ranges(row_ranges), _total_rows(total_rows), _ctz(ctz), _io_ctx(io_ctx) {} virtual ~ColumnReader() = default; virtual Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) = 0; @@ -198,7 +204,7 @@ class ColumnReader { return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } - static Status create(io::FileReaderSPtr file, FieldSchema* field, + static Status create(io::FileReaderSPtr file, NativeFieldSchema* field, const tparquet::RowGroup& row_group, const RowRanges& row_ranges, const cctz::time_zone* ctz, io::IOContext* io_ctx, std::unique_ptr& reader, size_t max_buf_size, @@ -220,13 +226,13 @@ class ColumnReader { virtual void reset_filter_map_index() = 0; - FieldSchema* get_field_schema() const { return _field_schema; } + NativeFieldSchema* get_field_schema() const { return _field_schema; } void set_column_in_nested() { _in_nested = true; } protected: void _generate_read_ranges(RowRange page_row_range, RowRanges* result_ranges) const; - FieldSchema* _field_schema = nullptr; + NativeFieldSchema* _field_schema = nullptr; const RowRanges& _row_ranges; size_t _total_rows = 0; const cctz::time_zone* _ctz = nullptr; @@ -254,11 +260,11 @@ class ScalarColumnReader : public ColumnReader { _chunk_meta(chunk_meta), _offset_index(offset_index) {} ~ScalarColumnReader() override { close(); } - Status init(io::FileReaderSPtr file, FieldSchema* field, size_t max_buf_size, + Status init(io::FileReaderSPtr file, NativeFieldSchema* field, size_t max_buf_size, RuntimeState* state, const std::string& page_cache_file_key, const ParquetReaderCompat& compat, bool enable_strict_mode); Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override; Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, @@ -394,9 +400,9 @@ class ArrayColumnReader : public ColumnReader { io::IOContext* io_ctx) : ColumnReader(row_ranges, total_rows, ctz, io_ctx) {} ~ArrayColumnReader() override { close(); } - Status init(std::unique_ptr element_reader, FieldSchema* field); + Status init(std::unique_ptr element_reader, NativeFieldSchema* field); Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override; Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, @@ -431,9 +437,9 @@ class MapColumnReader : public ColumnReader { ~MapColumnReader() override { close(); } Status init(std::unique_ptr key_reader, - std::unique_ptr value_reader, FieldSchema* field); + std::unique_ptr value_reader, NativeFieldSchema* field); Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override; Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, @@ -479,9 +485,9 @@ class StructColumnReader : public ColumnReader { ~StructColumnReader() override { close(); } Status init(std::unordered_map>&& child_readers, - FieldSchema* field); + NativeFieldSchema* field); Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override; Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, @@ -546,7 +552,7 @@ class StructColumnReader : public ColumnReader { class SkipReadingReader : public ColumnReader { public: SkipReadingReader(const RowRanges& row_ranges, size_t total_rows, const cctz::time_zone* ctz, - io::IOContext* io_ctx, FieldSchema* field_schema) + io::IOContext* io_ctx, NativeFieldSchema* field_schema) : ColumnReader(row_ranges, total_rows, ctz, io_ctx) { _field_schema = field_schema; // Use inherited member from base class VLOG_DEBUG << "[ParquetReader] Created SkipReadingReader for field: " @@ -554,7 +560,7 @@ class SkipReadingReader : public ColumnReader { } Status read_column_data(ColumnPtr& doris_column, const DataTypePtr& type, - const std::shared_ptr& root_node, + const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override { VLOG_DEBUG << "[ParquetReader] SkipReadingReader::read_column_data for field: " @@ -595,7 +601,7 @@ class SkipReadingReader : public ColumnReader { static std::unique_ptr create_unique(const RowRanges& row_ranges, size_t total_rows, cctz::time_zone* ctz, io::IOContext* io_ctx, - FieldSchema* field_schema) { + NativeFieldSchema* field_schema) { return std::make_unique(row_ranges, total_rows, ctz, io_ctx, field_schema); } diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp index 831b6268b6310c..53fb11a09ee87f 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.cpp @@ -25,7 +25,9 @@ Status FixLengthPlainDecoder::decode_fixed_values(size_t num_values, ParquetFixedValueConsumer& consumer) { const size_t byte_size = num_values * static_cast(_type_length); if (UNLIKELY(_offset > _data->size || byte_size > _data->size - _offset)) { - return Status::IOError("Out-of-bounds access in Parquet plain decoder"); + // Truncated PLAIN pages retain the public error keyword used by corrupt-file regression + // checks, while the bounds check still prevents the native decoder from reading past data. + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); } RETURN_IF_ERROR(consumer.consume(reinterpret_cast(_data->data) + _offset, num_values, static_cast(_type_length))); @@ -43,7 +45,7 @@ Status FixLengthPlainDecoder::decode_selected_fixed_values(const ParquetSelectio } const size_t input_bytes = selection.total_values * value_width; if (UNLIKELY(_offset > _data->size || input_bytes > _data->size - _offset)) { - return Status::IOError("Out-of-bounds access in Parquet plain selection decoder"); + return Status::IOError("Unexpected end of stream in Parquet plain selection decoder"); } const auto* values = reinterpret_cast(_data->data) + _offset; _offset += input_bytes; @@ -57,7 +59,7 @@ Status FixLengthPlainDecoder::skip_values(size_t num_values) { DORIS_CHECK(_type_length > 0); const size_t value_width = static_cast(_type_length); if (UNLIKELY(_offset > _data->size || num_values > (_data->size - _offset) / value_width)) { - return Status::IOError("Out-of-bounds access in parquet data decoder"); + return Status::IOError("Unexpected end of stream in Parquet plain decoder"); } _offset += num_values * value_width; return Status::OK(); diff --git a/be/src/format_v2/parquet/reader/native/level_reader.cpp b/be/src/format_v2/parquet/reader/native/level_reader.cpp index 4b9aa202cd4f97..01efbcae9b6233 100644 --- a/be/src/format_v2/parquet/reader/native/level_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/level_reader.cpp @@ -20,7 +20,7 @@ #include #include -#include "format/parquet/schema_desc.h" +#include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/fs/buffered_reader.h" #include "io/fs/tracing_file_reader.h" @@ -40,9 +40,9 @@ class LevelReader::Impl { template class LevelReaderImpl final : public LevelReader::Impl { public: - LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, FieldSchema* field, - size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key, + LevelReaderImpl(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, + io::IOContext* io_ctx, bool enable_page_cache, std::string page_cache_file_key, ParquetReaderCompat compat) : _file(std::move(file)), _column_chunk(std::move(column_chunk)), @@ -155,7 +155,7 @@ class LevelReaderImpl final : public LevelReader::Impl { io::FileReaderSPtr _file; tparquet::ColumnChunk _column_chunk; - FieldSchema* _field = nullptr; + NativeFieldSchema* _field = nullptr; size_t _total_rows = 0; size_t _max_buffer_size = 0; io::IOContext* _io_ctx = nullptr; @@ -168,7 +168,7 @@ class LevelReaderImpl final : public LevelReader::Impl { }; Status LevelReader::create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, - FieldSchema* field, size_t total_rows, size_t max_buffer_size, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, bool enable_page_cache, const std::string& page_cache_file_key, const ParquetReaderCompat& compat, diff --git a/be/src/format_v2/parquet/reader/native/level_reader.h b/be/src/format_v2/parquet/reader/native/level_reader.h index 814c7ffa0ed804..11117d25dcb252 100644 --- a/be/src/format_v2/parquet/reader/native/level_reader.h +++ b/be/src/format_v2/parquet/reader/native/level_reader.h @@ -28,12 +28,9 @@ #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "io/fs/file_reader_writer_fwd.h" -namespace doris { -struct FieldSchema; -namespace io { +namespace doris::io { struct IOContext; -} -} // namespace doris +} // namespace doris::io namespace doris::format::parquet::native { @@ -50,7 +47,7 @@ class LevelReader { class Impl; static Status create(io::FileReaderSPtr file, tparquet::ColumnChunk column_chunk, - FieldSchema* field, size_t total_rows, size_t max_buffer_size, + NativeFieldSchema* field, size_t total_rows, size_t max_buffer_size, io::IOContext* io_ctx, bool enable_page_cache, const std::string& page_cache_file_key, const ParquetReaderCompat& compat, std::unique_ptr* reader); diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 55127ae8d47f6b..487bb747c9ab49 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -93,15 +93,16 @@ DataTypePtr projected_type(const ParquetColumnSchema& schema, return nullptr; } -const FieldSchema* find_child_field(const FieldSchema& parent, const ParquetColumnSchema& child) { - auto field_it = std::ranges::find_if(parent.children, [&](const FieldSchema& field) { +const NativeFieldSchema* find_child_field(const NativeFieldSchema& parent, + const ParquetColumnSchema& child) { + auto field_it = std::ranges::find_if(parent.children, [&](const NativeFieldSchema& field) { return (child.parquet_field_id >= 0 && field.field_id == child.parquet_field_id) || field.name == child.name; }); return field_it == parent.children.end() ? nullptr : &*field_it; } -void collect_physical_subtree_ids(const FieldSchema& field, std::set* ids) { +void collect_physical_subtree_ids(const NativeFieldSchema& field, std::set* ids) { DORIS_CHECK(ids != nullptr); ids->insert(field.get_column_id()); for (const auto& child : field.children) { @@ -111,7 +112,7 @@ void collect_physical_subtree_ids(const FieldSchema& field, std::set* void collect_projected_ids(const ParquetColumnSchema& schema, const format::LocalColumnIndex* projection, - const FieldSchema& native_field, std::set* ids) { + const NativeFieldSchema& native_field, std::set* ids) { DORIS_CHECK(ids != nullptr); if (!format::is_partial_projection(projection)) { return; @@ -121,7 +122,7 @@ void collect_projected_ids(const ParquetColumnSchema& schema, return child->local_id == child_projection.local_id(); }); DORIS_CHECK(schema_it != schema.children.end()); - const FieldSchema* child_field = find_child_field(native_field, **schema_it); + const NativeFieldSchema* child_field = find_child_field(native_field, **schema_it); DORIS_CHECK(child_field != nullptr); if (format::is_full_projection(&child_projection)) { // A full child path is a request for its complete physical subtree. Keeping only the @@ -192,7 +193,7 @@ Status NativeColumnReader::create( return Status::InvalidArgument("Invalid native parquet top-level column id {} for {}", column_schema.local_id, column_schema.name); } - auto* field = const_cast(native_schema.get_column(column_schema.local_id)); + auto* field = const_cast(native_schema.get_column(column_schema.local_id)); DORIS_CHECK(field != nullptr); if (field->name != column_schema.name && !(field->field_id >= 0 && field->field_id == column_schema.parquet_field_id)) { @@ -202,9 +203,8 @@ Status NativeColumnReader::create( } auto type = projected_type(column_schema, projection); - std::shared_ptr schema_node; - RETURN_IF_ERROR(TableSchemaChangeHelper::BuildTableInfoUtil::by_parquet_name(type, *field, - schema_node)); + std::shared_ptr schema_node; + RETURN_IF_ERROR(build_native_schema_node(type, column_schema, &schema_node)); std::set projected_ids; collect_projected_ids(column_schema, projection, *field, &projected_ids); @@ -220,7 +220,7 @@ Status NativeColumnReader::create( Status NativeColumnReader::init( io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, - FieldSchema* field, std::shared_ptr schema_node, + NativeFieldSchema* field, std::shared_ptr schema_node, std::set projected_column_ids, const std::vector& selected_ranges, const std::unordered_map& offset_indexes, const cctz::time_zone* timezone, io::IOContext* io_ctx, RuntimeState* runtime_state, diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 8dcd3590808f13..ad287ed8cec308 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -29,8 +29,8 @@ #include #include "format/parquet/parquet_common.h" -#include "format/table/table_schema_change_helper.h" #include "format_v2/column_data.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/reader/column_reader.h" #include "format_v2/parquet/reader/native/column_reader.h" @@ -97,7 +97,7 @@ class NativeColumnReader final : public ParquetColumnReader { ParquetColumnReaderProfile profile); Status init(io::FileReaderSPtr file, const NativeParquetMetadata* metadata, int row_group_id, - FieldSchema* field, std::shared_ptr schema_node, + NativeFieldSchema* field, std::shared_ptr schema_node, std::set projected_column_ids, const std::vector& selected_ranges, const std::unordered_map& offset_indexes, @@ -118,7 +118,7 @@ class NativeColumnReader final : public ParquetColumnReader { std::set _projected_column_ids; std::set _filter_column_ids; const std::unordered_map* _offset_indexes = nullptr; - std::shared_ptr _schema_node; + std::shared_ptr _schema_node; std::unique_ptr _native_reader; std::unique_ptr _page_cache_runtime_state; std::vector _selected_ranges; diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 08d7e61ea1caa8..ab99f94989a1b7 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -130,8 +130,8 @@ class ScriptedColumnReader final : public ColumnReader { _def_levels(std::move(def_levels)) {} Status read_column_data(ColumnPtr& column, const DataTypePtr&, - const std::shared_ptr&, FilterMap&, - size_t, size_t* read_rows, bool* eof, bool, int64_t = -1) override { + const std::shared_ptr&, FilterMap&, size_t, + size_t* read_rows, bool* eof, bool, int64_t = -1) override { if (_used) { *read_rows = 0; *eof = true; @@ -262,7 +262,7 @@ Status load_scripted_page(tparquet::PageHeader header, const std::vector(6)); + NativeFieldSchema int96_field; + int96_field.physical_type = tparquet::Type::INT96; + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, false)); + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test(int96_field, datetime_type, true)); + + NativeFieldSchema utc_field; + utc_field.physical_type = tparquet::Type::INT64; + utc_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + utc_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + utc_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(utc_field, datetime_type, false)); + + NativeFieldSchema converted_utc_field; + converted_utc_field.physical_type = tparquet::Type::INT64; + converted_utc_field.parquet_schema.__set_converted_type( + tparquet::ConvertedType::TIMESTAMP_MICROS); + EXPECT_TRUE(preserves_timestamp_conversion_default_for_test(converted_utc_field, datetime_type, + false)); + + NativeFieldSchema local_field = utc_field; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + EXPECT_FALSE( + preserves_timestamp_conversion_default_for_test(local_field, datetime_type, false)); + + NativeFieldSchema integer_field; + integer_field.physical_type = tparquet::Type::INT64; + EXPECT_FALSE(preserves_timestamp_conversion_default_for_test( + integer_field, make_nullable(std::make_shared()), false)); +} + +TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { + DataTypePtr datetime_type = make_nullable(std::make_shared(6)); + NativeFieldSchema local_field; + local_field.physical_type = tparquet::Type::INT64; + local_field.parquet_schema.__set_logicalType(tparquet::LogicalType()); + local_field.parquet_schema.logicalType.__set_TIMESTAMP(tparquet::TimestampType()); + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(false); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + const uint64_t invalid_datetime = 0; + column->insert_data(reinterpret_cast(&invalid_datetime), sizeof(invalid_datetime)); + IColumn::Filter null_map; + null_map.resize_fill(1, 0); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 1); + + null_map[0] = 0; + local_field.parquet_schema.logicalType.TIMESTAMP.__set_isAdjustedToUTC(true); + mark_local_timestamp_defaults_for_test(local_field, datetime_type, false, *column, &null_map, + 0); + EXPECT_EQ(null_map[0], 0); +} + TEST(ParquetV2NativeDecoderTest, SparsePlainAndBooleanDecodeOnceAndPreserveCursor) { const ParquetSelection selection { .total_values = 7, @@ -948,6 +1005,21 @@ TEST(ParquetV2NativeDecoderTest, DeltaFixedWidthValidatesFilteredAndSkippedValue EXPECT_TRUE(decoder->skip_values(2).is()); } +TEST(ParquetV2NativeDecoderTest, TruncatedFixedWidthPlainKeepsPublicErrorKeyword) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::Encoding::PLAIN, decoder) + .ok()); + decoder->set_type_length(4); + char truncated[] = {'a', 'b', 'c'}; + Slice slice(truncated, sizeof(truncated)); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureFixedConsumer consumer; + const auto status = decoder->decode_fixed_values(1, consumer); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Unexpected end of stream"), std::string::npos); +} + TEST(ParquetV2NativeDecoderTest, DeltaSkipRequiresTheRequestedValueCount) { const std::vector integers {7}; auto int_descriptor = descriptor(::parquet::Type::INT32); @@ -1066,7 +1138,7 @@ Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t va auto value_type = make_nullable(std::make_shared()); auto map_type = std::make_shared(key_type, value_type); ColumnPtr column = map_type->create_column(); - FieldSchema field; + NativeFieldSchema field; field.name = "m"; field.data_type = map_type; field.definition_level = 1; @@ -1082,9 +1154,8 @@ Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t va MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); - auto root = std::make_shared( - TableSchemaChangeHelper::ConstNode::get_instance(), - TableSchemaChangeHelper::ConstNode::get_instance()); + auto root = std::make_shared(std::make_shared(), + std::make_shared()); FilterMap filter; size_t read_rows = 0; bool eof = false; @@ -1103,10 +1174,9 @@ TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { read_scripted_map(int_type, 2, 1, 2, 1, {0, 0}, {0}, true).is()); } -TEST(ParquetV2NativeDecoderTest, MapReaderRejectsNullKeys) { +TEST(ParquetV2NativeDecoderTest, MapReaderPreservesNullableKeysWrittenByDoris) { auto nullable_key = make_nullable(std::make_shared()); - EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true) - .is()); + EXPECT_TRUE(read_scripted_map(nullable_key, 1, 1, 1, 1, {0}, {0}, true).ok()); } TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { @@ -1114,7 +1184,7 @@ TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { auto struct_type = std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); ColumnPtr column = struct_type->create_column(); - FieldSchema field; + NativeFieldSchema field; field.name = "s"; field.data_type = struct_type; field.children.resize(2); @@ -1128,9 +1198,9 @@ TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShortSibling) { std::vector {0}); StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); ASSERT_TRUE(reader.init(std::move(children), &field).ok()); - auto root = std::make_shared(); - root->add_children("a", "a", TableSchemaChangeHelper::ConstNode::get_instance()); - root->add_children("b", "b", TableSchemaChangeHelper::ConstNode::get_instance()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); FilterMap filter; size_t read_rows = 0; bool eof = false; @@ -1144,7 +1214,7 @@ TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) auto struct_type = std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); ColumnPtr column = struct_type->create_column(); - FieldSchema field; + NativeFieldSchema field; field.name = "s"; field.repetition_level = 1; field.children.resize(2); @@ -1158,9 +1228,9 @@ TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) 2, 3, true, std::vector {0, 0, 1}, std::vector {0, 0, 0}); StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); ASSERT_TRUE(reader.init(std::move(children), &field).ok()); - auto root = std::make_shared(); - root->add_children("a", "a", TableSchemaChangeHelper::ConstNode::get_instance()); - root->add_children("b", "b", TableSchemaChangeHelper::ConstNode::get_instance()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); FilterMap filter; size_t read_rows = 0; bool eof = false; @@ -1250,7 +1320,7 @@ TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMis chunk.meta_data.__set_num_values(2); chunk.meta_data.__set_total_compressed_size(bytes.size()); chunk.meta_data.__set_data_page_offset(0); - FieldSchema field; + NativeFieldSchema field; field.physical_type = tparquet::Type::INT32; field.repetition_level = 0; field.definition_level = 0; @@ -1413,6 +1483,16 @@ TEST(ParquetV2NativeDecoderTest, ColumnChunkRangeRejectsSignedOverflowAndBoundsL ASSERT_TRUE(compute_column_chunk_range(metadata, 35, true, &range).ok()); EXPECT_EQ(range.offset, 10); EXPECT_EQ(range.length, 25); + + metadata.__set_dictionary_page_offset(0); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 10); + EXPECT_EQ(range.length, 20); + + metadata.__set_dictionary_page_offset(5); + ASSERT_TRUE(compute_column_chunk_range(metadata, 35, false, &range).ok()); + EXPECT_EQ(range.offset, 5); + EXPECT_EQ(range.length, 20); } TEST(ParquetV2NativeDecoderTest, OffsetIndexValidationRejectsBackwardAndOverlappingLocations) { @@ -1488,7 +1568,7 @@ TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingData chunk.meta_data.__set_num_values(1); chunk.meta_data.__set_total_compressed_size(bytes.size()); chunk.meta_data.__set_data_page_offset(0); - FieldSchema field; + NativeFieldSchema field; field.physical_type = tparquet::Type::INT32; ParquetPageReadContext context(false, ""); ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, @@ -1601,6 +1681,14 @@ TEST(ParquetV2NativeDecoderTest, UncompressedV2PageCachePayloadIsAlwaysDecompres EXPECT_TRUE(should_cache_decompressed(&header, metadata)); } +TEST(ParquetV2NativeDecoderTest, CacheDisabledPagesDoNotPrepareCopiedPayload) { + EXPECT_FALSE(can_prepare_page_cache_payload(false, false, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, true, true, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, false, true)); + EXPECT_FALSE(can_prepare_page_cache_payload(true, false, true, false)); + EXPECT_TRUE(can_prepare_page_cache_payload(true, false, true, true)); +} + TEST(ParquetV2NativeDecoderTest, OversizedNestedBatchScratchUsesIdleBatchHysteresis) { ::doris::RowRanges row_ranges; tparquet::ColumnChunk chunk; diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index 940f94a373a013..7176e1a054e77a 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -173,6 +173,19 @@ TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) EXPECT_EQ(valid_ranges[1].size, 60); } +TEST(ParquetPageCacheRangeTest, SerializedIndexesAreBoundedIndividuallyAndWhenCoalesced) { + constexpr size_t file_size = 1ULL << 30; + const auto budget = detail::MAX_SERIALIZED_PARQUET_INDEX_BYTES; + + EXPECT_TRUE(detail::is_serialized_index_range_safe(file_size, 0, budget)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, 0, budget + 1)); + EXPECT_FALSE(detail::is_serialized_index_range_safe(file_size, -1, 1)); + + EXPECT_TRUE(detail::is_serialized_index_span_safe(100, 100 + budget)); + // Individually small adjacent indexes must not combine into one unbounded allocation. + EXPECT_FALSE(detail::is_serialized_index_span_safe(100, 101 + budget)); +} + TEST(ParquetPageCacheRangeTest, AveragePrefetchRangeSizeUsesOnlyValidRanges) { const std::vector ranges = { {0, 512}, diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 77df0b43735725..a90925f0eb58ec 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -2024,6 +2024,13 @@ TEST_F(NewParquetReaderTest, ReadPredicateAndNonPredicateColumnsWithSelection) { EXPECT_EQ(profile.get_counter("RawRowsRead")->value(), ROW_COUNT); EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 3); EXPECT_EQ(profile.get_counter("RowsFilteredByConjunct")->value(), 2); + TRuntimeProfileTree profile_tree; + profile.to_thrift(&profile_tree); + ASSERT_FALSE(profile_tree.nodes.empty()); + const auto parquet_children = + profile_tree.nodes.front().child_counters_map.find("ParquetReader"); + ASSERT_NE(parquet_children, profile_tree.nodes.front().child_counters_map.end()); + EXPECT_TRUE(parquet_children->second.contains("RowsFilteredByConjunct")); EXPECT_EQ(profile.get_counter("TotalBatches")->value(), 1); EXPECT_EQ(profile.get_counter("DenseBatches")->value(), 0); EXPECT_EQ(profile.get_counter("SelectedBatches")->value(), 1); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 9f78735bfb560a..9c87572866cb8f 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -27,6 +27,8 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/primitive_type.h" +#include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/parquet_column_schema.h" namespace doris::format::parquet { @@ -78,6 +80,49 @@ TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { EXPECT_TRUE(fields[1]->type->is_nullable()); } +TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement protocol; + protocol.__set_name("protocol"); + protocol.__set_num_children(2); + protocol.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + protocol.__set_field_id(10); + + tparquet::SchemaElement min_reader; + min_reader.__set_name("minReaderVersion"); + min_reader.__set_type(tparquet::Type::INT32); + min_reader.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + min_reader.__set_field_id(11); + + tparquet::SchemaElement min_writer = min_reader; + min_writer.__set_name("minWriterVersion"); + min_writer.__set_field_id(12); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, protocol, min_reader, min_writer}).ok()); + native_schema.assign_ids(); + + std::vector> fields; + ASSERT_TRUE(build_parquet_column_schema(native_schema, &fields).ok()); + ASSERT_EQ(fields.size(), 1); + EXPECT_EQ(fields[0]->name, "protocol"); + EXPECT_EQ(fields[0]->parquet_field_id, 10); + ASSERT_EQ(fields[0]->children.size(), 2); + EXPECT_EQ(fields[0]->children[0]->name, "minReaderVersion"); + EXPECT_EQ(fields[0]->children[0]->leaf_column_id, 0); + EXPECT_EQ(fields[0]->children[1]->name, "minWriterVersion"); + EXPECT_EQ(fields[0]->children[1]->leaf_column_id, 1); + + std::shared_ptr mapping; + ASSERT_TRUE(build_native_schema_node(fields[0]->type, *fields[0], &mapping).ok()); + EXPECT_TRUE(mapping->has_child("minReaderVersion")); + EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); + ASSERT_NE(mapping->child("minReaderVersion"), nullptr); +} + TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { const auto fields = build_fields({ ::parquet::schema::PrimitiveNode::Make( @@ -532,4 +577,53 @@ TEST(ParquetSchemaTest, LogicalUtcTimeIsPreservedForProjection) { EXPECT_EQ(remove_nullable(fields[0]->children[1]->type)->get_primitive_type(), TYPE_BIGINT); } +TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsRejected) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement adjusted_time; + adjusted_time.__set_name("time_ms"); + adjusted_time.__set_type(tparquet::Type::INT32); + adjusted_time.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + adjusted_time.__set_logicalType(tparquet::LogicalType()); + adjusted_time.logicalType.__set_TIME(tparquet::TimeType()); + adjusted_time.logicalType.TIME.__set_isAdjustedToUTC(true); + adjusted_time.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + adjusted_time.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + + NativeFieldDescriptor native_schema; + ASSERT_TRUE(native_schema.parse_from_thrift({root, adjusted_time}).ok()); + native_schema.assign_ids(); + std::vector> fields; + const auto status = build_parquet_column_schema(native_schema, &fields); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), + std::string::npos); +} + +TEST(ParquetSchemaTest, NativeGroupEnumLogicalTypeIsRejected) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + + tparquet::SchemaElement enum_group; + enum_group.__set_name("bad_enum_group"); + enum_group.__set_num_children(1); + enum_group.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + enum_group.__set_logicalType(tparquet::LogicalType()); + enum_group.logicalType.__set_ENUM(tparquet::EnumType()); + + tparquet::SchemaElement child; + child.__set_name("value"); + child.__set_type(tparquet::Type::BYTE_ARRAY); + child.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + + NativeFieldDescriptor native_schema; + const auto status = native_schema.parse_from_thrift({root, enum_group, child}); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("Logical type Enum cannot be applied to group node"), + std::string::npos); +} + } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index 4bb9399045ea0c..0788f09b736cfc 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -832,6 +832,40 @@ TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsu EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); } +TEST(ParquetStatisticsPruningTest, NativeContradictoryNullMetadataFallsBackForNullPredicates) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), + {int32_array({1})}); + auto reader = make_reader(table, 1, false, true); + auto schema = build_file_schema(*reader); + + auto make_indexes = [](int64_t null_count) { + format::parquet::NativeParquetPageIndex indexes; + indexes.column_index.__set_null_pages({true}); + indexes.column_index.__set_null_counts({null_count}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(1); + location.__set_first_row_index(0); + indexes.offset_index.__set_page_locations({location}); + return std::unordered_map { + {0, std::move(indexes)}}; + }; + + for (const auto op : {Int32ZoneMapExpr::Op::IS_NULL, Int32ZoneMapExpr::Op::IS_NOT_NULL}) { + std::vector selected; + const int64_t null_count = op == Int32ZoneMapExpr::Op::IS_NULL ? 0 : -1; + ASSERT_TRUE( + format::parquet::select_row_group_ranges_by_native_page_index( + make_indexes(null_count), schema, + request_with_zonemap_conjunct(std::make_shared(0, op)), 1, + &selected, nullptr, nullptr) + .ok()); + ASSERT_EQ(selected.size(), 1); + EXPECT_EQ(selected[0].start, 0); + EXPECT_EQ(selected[0].length, 1); + } +} + TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), {string_array({"alpha", "beta", "gamma", "omega"})}); diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index d41249dd8af90b..67c7f226cd6401 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -120,9 +120,10 @@ format-specific checklist when reviewing Parquet or ORC. - V2 must instantiate only readers and decoders under `be/src/format_v2/parquet/`; calls into the v1 `ParquetColumnReader` or edits under `be/src/format/parquet/` are review blockers. -- Footer parsing, schema-ID assignment, retained serialized bytes, and the cached metadata payload - must also be v2-owned. Reusing a stable base file identity is allowed, but require a v2 cache type - discriminator so v1/v2 metadata objects can never be cross-cast. +- Footer parsing, schema-ID assignment, and the cached native metadata tree must also be v2-owned. + Reusing a stable base file identity is allowed, but require a v2 cache type discriminator so + v1/v2 metadata objects can never be cross-cast. Production planning must not retain or rebuild an + Arrow `FileMetaData` tree from the serialized footer. - Trace the hot path as `ColumnReader -> Decoder span/cursor API -> DataTypeSerDe -> Doris Column`. Decoder must not accept a Doris column or target type, and the path must not create Arrow arrays, builders, `DecodedColumnView`, or another decoded leaf batch. @@ -208,9 +209,9 @@ format-specific checklist when reviewing Parquet or ORC. - Keep Parquet decimals in a source-width or wider intermediate until exact scaling, target precision, and overflow checks succeed. Scale-down with a non-zero remainder is a conversion failure; plain and dictionary integer/binary paths must narrow only afterward. -- For cold small-file tests, separate footer I/O/Thrift parse from Arrow metadata adaptation. V2 may - retain the already-read serialized footer to avoid serializing the same Thrift object again; v1 - opens must not retain those bytes by default. +- For cold small-file tests, separate footer I/O/Thrift parse from native schema and index planning. + V2 must not retain serialized footer bytes after the native metadata tree is initialized; v1 opens + remain independent. - For HTTP Parquet objects at or below `in_memory_file_size`, v2 stages the complete object from byte zero before native page access. Verify both cold and footer-cache-hit scans: some Range servers accept the capability probe but return HTTP 200 for a near-EOF overlong range, so warm @@ -257,8 +258,8 @@ format-specific checklist when reviewing Parquet or ORC. the legacy consumer fallback for non-string logical types. - Production PageIndex planning must consume native Compact Thrift ColumnIndex/OffsetIndex objects. Coalesce adjacent serialized index ranges and transfer validated OffsetIndexes into execution so - the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; any remaining - Arrow metadata adapter must expose its time and retained bytes until it is removed. + the Row Group does not read them twice. Arrow PageIndexReader is a test oracle; a production Arrow + metadata adapter or rebuilt `FileMetaData` tree is a review blocker. - Fixed-width conversion fast paths may remove row branches only when the source domain provably fits the target domain. Narrowing, timestamp, decimal scaling, strict rollback, and non-strict NULL marking must retain corrupt-value tests. diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index d21d4688f2fb00..5ad62f04a60532 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -79,14 +79,12 @@ the production v2 path never instantiates the v1 `ParquetColumnReader`: | Native column reader | `NativeColumnReader` is persistent for one top-level column and Row Group. It owns selection/filter/dictionary scratch and drives the native decoder directly into the final Doris column. | | Complex reconstruction | Choose a Dremel shape-owning leaf per requested parent-row range; derive parent offsets/nulls once from it and keep sibling leaf streams aligned to the same parent rows. | | Metadata and planning | Replace Arrow footer/schema/Row Group metadata dependencies with native Thrift-derived objects while preserving the existing planner, index, cache, and split contracts. | -| Compatibility removal | Remove Arrow data-read adapters after type/encoding/page/writer compatibility and performance gates pass. Production v2 never falls back from a selected native reader to Arrow; an unsupported combination returns an explicit error. | +| Compatibility removal | Production v2 has no Arrow data-read or metadata adapter and never falls back to the v1 reader; unsupported combinations return explicit errors. Arrow remains only in test oracles and fixture writers. | Data-page value scans and levels-only aggregate scans no longer use Arrow `RecordReader`, arrays, or -builders. V2 parses and owns the Thrift footer and physical schema; a lazy Arrow metadata adapter -remains only for planner consumers that have not yet migrated. It is not a runtime fallback: after -a column selects the native reader, decode errors are returned directly. The production target -eventually removes that remaining planner adapter; tests may also use Arrow as a fixture writer or -oracle. +builders. V2 parses and owns the Thrift footer, schema parser, field-ID assignment, physical schema, +statistics, dictionary metadata, bloom filters, and page indexes. Production planning never builds +an Arrow metadata tree; Arrow is limited to fixture writers and differential test oracles. All new integration remains in `be/src/format_v2/parquet/`. V1 is kept unchanged as the correctness and performance control. Compatibility is demonstrated through differential tests and the explicit @@ -156,14 +154,12 @@ sequenceDiagram delete conjuncts, and local column-position mappings. - **RowGroupReadPlan:** Records the Row Group, its file-global starting row, `selected_ranges` produced by page-index pruning, and the `page_skip_plan` for each leaf column. -- **ParquetFileContext:** Adapts Doris FileReader to the active metadata/data stream interface and - owns Page Cache, FileCache prefetch, and MergeRange routing. During migration this may still - expose an Arrow adapter for metadata consumers, but native page decoding reads the same stable - Doris byte ranges without producing Arrow arrays. The immutable native footer owns one - thread-safe, lazily constructed Arrow metadata adapter at the footer-cache lifecycle, so repeated - v2 opens neither re-read the footer nor serialize and parse the same metadata again. - -For every selected Row Group, dictionary/index probes finish on the metadata adapter first. The +- **ParquetFileContext:** Adapts Doris FileReader to the native metadata/data stream interface and + owns Page Cache, FileCache prefetch, and MergeRange routing. Its immutable footer cache entry owns + the V2 `NativeFieldDescriptor` tree directly; repeated V2 opens neither re-read the footer nor + serialize and parse another metadata representation. + +For every selected Row Group, native dictionary/index probes finish on the cached metadata tree first. The scheduler then computes projected physical Column Chunk ranges and installs one shared native `MergeRangeFileReader` when their average size is below v1's small-I/O threshold. All predicate and lazy output readers share that wrapper; their internal `BufferedFileStreamReader` prefetch is @@ -657,7 +653,7 @@ flowchart TB | Mechanism | Cached or optimized object | Lifecycle and key | Problem addressed | | --- | --- | --- | --- | -| Footer metadata cache | V2-owned immutable Thrift metadata, v2 physical schema, and the serialized footer bytes already fetched from storage | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, unsafe cross-version cache casts, and Thrift re-serialization before Arrow metadata adaptation | +| Footer metadata cache | V2-owned immutable Thrift metadata and native physical schema tree | Stable base file identity plus a v2 type discriminator and schema-affecting mapping options | Avoid repeated footer I/O, Thrift parsing, schema construction, and unsafe cross-version cache casts | | Small HTTP object staging | Complete object bytes for files at or below `in_memory_file_size` | Per-reader v2 wrapper; loaded once from byte zero and released with the file context | Collapse cold small-file requests and keep footer-cache-hit scans independent of server-specific near-EOF Range behavior | | FileCache | Remote file blocks | Related to filesystem/path and file version; may hit locally or through a peer | Avoid repeated object-storage access and support background prefetch | | Parquet Page Cache | Serialized bytes within registered Column Chunk ranges | Stable file key depends on path, mtime/version, and file size; disabled when mtime is unreliable | Reduce repeated page reads and support exact/subrange coverage | @@ -673,15 +669,13 @@ decoder, and scratch state remains per reader. Path-only keys are insufficient. short files, unsupported metadata, and schema-affecting option changes cannot populate or reuse a successful entry. No change to the v1 parser or metadata cache behavior is required. -On a v2 cold miss, the parser retains the exact serialized footer bytes already in memory, and the -lazy Arrow planner adapter parses those bytes directly instead of serializing the Thrift object -again. The adapter is cached inside the v2 footer-object lifecycle; it is never the cache payload -type and its lifetime does not enter the native decoder. Production ColumnIndex/OffsetIndex planning reads and -parses Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and the -validated OffsetIndex objects are transferred into Row Group execution instead of being fetched a -second time. The Arrow PageIndexReader path remains only as a test oracle. Until the remaining -row-group metadata planner adapter is removed, its construction cost and retained tree size are -visible as `ArrowMetadataAdapterTime/Bytes`. +On a v2 cold miss, the parser deserializes the footer once into the V2-owned Thrift object and builds +the native schema tree directly from `SchemaElement` into V2's `NativeFieldDescriptor`. Row-group statistics, +dictionary and Bloom metadata planning consume the same native tree; no Arrow `FileMetaData` tree +or serialized-footer copy is retained. Production ColumnIndex/OffsetIndex planning reads and parses +Compact Thrift indexes natively; adjacent per-leaf serialized ranges are coalesced, and validated +OffsetIndex objects are transferred into Row Group execution instead of being fetched a second +time. Arrow metadata and PageIndexReader paths remain only as test oracles. Before the footer lookup, v2 wraps bounded HTTP Parquet objects in an in-memory reader. The wrapper loads from byte zero on its first physical access, whether that access is a cold footer read or a @@ -711,8 +705,8 @@ Chunks from surviving Row Groups are registered, limiting pollution and key coun may be prefetched into FileCache. - After dictionary probing, small projected chunks share one Row-Group-scoped MergeRangeFileReader on the native data-page path. Large chunks and in-memory files keep the base - reader. The remaining Arrow metadata/index adapter has an independent wrapper and never owns the - native decoder's stream cursor. + reader. Native metadata/index reads use explicit immutable ranges and never own a decoder stream + cursor. - With row-level filters, prefetch predicate columns first. Prefetch non-predicate columns only after at least one row survives, avoiding unnecessary bandwidth. @@ -863,7 +857,7 @@ flowchart TD | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | | Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | | Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | -| Metadata lifecycle | How much time was spent reading the footer, adapting remaining Arrow planner metadata (`ArrowMetadataAdapterTime/Bytes`), natively reading/parsing page indexes, and evaluating page-index predicates? | +| Metadata lifecycle | How much time was spent reading/parsing the native footer and schema tree, natively reading/parsing page indexes, and evaluating row-group/page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | | FileCache Profile | How many local/peer/remote bytes, waits, downloads, and hits occurred? | | Merge / request I/O | Were small reads merged, and were request count and read amplification reasonable? | diff --git a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out index 4fe42d7fcdcc77..bb74c79d5a6907 100644 --- a/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out +++ b/regression-test/data/external_table_p0/tvf/test_hdfs_parquet_group6.out @@ -808,10 +808,10 @@ true -- !test_98 -- \N \N \N -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 -abcDeFGhijkLmnOp \N 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 +abcDeFGhijkLmnOp 682.56 1212 -- !test_100 -- 1317017856 1 18752152 809291 1089176 19951117 3-MEDIUM 0 40 4801000 16034243 9 4368910 72015 3 19951228 RAIL Customer#018752152 q4gN2btSpiKXdN,6 ALGERIA 1 ALGERIA AFRICA 10-753-996-8708 MACHINERY Supplier#001089176 ROidEL1L6yeFsJqnUjD EGYPT 5 EGYPT MIDDLE EAST 14-807-108-7869 blanched gainsboro MFGR#4 MFGR#43 MFGR#433 brown MEDIUM BRUSHED STEEL 42 MED BAG @@ -866,10 +866,10 @@ abcDeFGhijkLmnOp \N 1212 -- !test_107 -- \N \N \N -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 -0x6162634465464768696A6B4C6D6E4F70 \N 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 +0x6162634465464768696A6B4C6D6E4F70 682.56 1212 -- !test_107_desc -- decimal_flba decimal(5,2) Yes false \N NONE diff --git a/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out b/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out index 1a2c37fe9dfe7a..5441e21cdf8dcb 100644 --- a/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out +++ b/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out @@ -144,7 +144,7 @@ PhysicalResultSink --NestedLoopJoin[INNER_JOIN] ----filter((pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) +----filter((cast(fkt_not_null.f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) ------PhysicalOlapScan[fkt_not_null] -- !res -- @@ -155,9 +155,9 @@ pk with filter that not same as fk -- !shape -- PhysicalResultSink --NestedLoopJoin[INNER_JOIN] -----filter((cast(p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) +----filter((cast(pkt.p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) +----filter((cast(fkt_not_null.f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) ------PhysicalOlapScan[fkt_not_null] -- !res -- @@ -167,7 +167,7 @@ simple_case -- !shape -- PhysicalResultSink ---filter(( not fk IS NULL)) +--filter(( not fkt.fk IS NULL)) ----PhysicalOlapScan[fkt] -- !res -- @@ -300,7 +300,7 @@ PhysicalResultSink --NestedLoopJoin[INNER_JOIN] ----filter((pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) +----filter((cast(fkt.f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) ------PhysicalOlapScan[fkt] -- !res -- @@ -311,9 +311,9 @@ pk with filter that not same as fk -- !shape -- PhysicalResultSink --NestedLoopJoin[INNER_JOIN] -----filter((cast(p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) +----filter((cast(pkt.p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) +----filter((cast(fkt.f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) ------PhysicalOlapScan[fkt] -- !res -- @@ -325,7 +325,7 @@ multi_table_join_with_pk_predicate PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((fkt_not_null.fk = fkt_not_null2.fk)) otherCondition=() ----hashJoin[INNER_JOIN] hashCondition=((pkt.pk = fkt_not_null.fk)) otherCondition=() -------filter((cast(p as DECIMALV3(38, 6)) = 1.000000)) +------filter((cast(pkt.p as DECIMALV3(38, 6)) = 1.000000)) --------PhysicalOlapScan[pkt] ------PhysicalOlapScan[fkt_not_null] ----PhysicalOlapScan[fkt_not_null(fkt_not_null2)] From 1cf1dc152c78bfbe7a2567a00a517d9c0c205356 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 18 Jul 2026 23:21:12 +0800 Subject: [PATCH 16/34] [fix](be) harden native Parquet metadata validation --- .../format_v2/parquet/native_schema_desc.cpp | 56 ++++- be/src/format_v2/parquet/native_schema_desc.h | 7 + .../parquet/parquet_file_context.cpp | 27 +++ be/src/format_v2/parquet/parquet_scan.cpp | 191 +++++++++++------ be/src/format_v2/parquet/parquet_scan.h | 13 +- .../format_v2/parquet/parquet_statistics.cpp | 72 ++++++- be/src/format_v2/parquet/parquet_statistics.h | 6 + .../native/block_split_bloom_filter.cpp | 116 +++++++++++ .../reader/native/block_split_bloom_filter.h | 54 +++++ .../reader/native/bool_plain_decoder.cpp | 1 - .../reader/native/bool_rle_decoder.cpp | 1 - .../reader/native/byte_array_plain_decoder.h | 1 - .../reader/native/column_chunk_reader.cpp | 18 ++ .../reader/native/column_chunk_reader.h | 3 +- .../parquet/reader/native/column_reader.cpp | 154 ++++++++++++++ .../parquet/reader/native/column_reader.h | 2 +- .../parquet/reader/native/common.cpp | 196 ++++++++++++++++++ .../format_v2/parquet/reader/native/common.h | 111 ++++++++++ .../reader/native/delta_bit_pack_decoder.h | 1 - .../reader/native/fix_length_plain_decoder.h | 5 - .../parquet/reader/native/level_decoder.cpp | 1 - .../parquet/reader/native/level_decoder.h | 2 +- .../parquet/reader/native/page_reader.cpp | 1 - .../parquet/reader/native/page_reader.h | 1 - .../parquet/reader/native_column_reader.cpp | 2 +- .../parquet/reader/native_column_reader.h | 1 - .../format_v2/parquet/native_decoder_test.cpp | 130 ++++++++++++ .../format_v2/parquet/parquet_scan_test.cpp | 58 ++++++ .../format_v2/parquet/parquet_schema_test.cpp | 80 +++++++ .../parquet/parquet_statistics_test.cpp | 27 +++ .../nereids_rules_p0/pkfk/eliminate_inner.out | 16 +- 31 files changed, 1248 insertions(+), 106 deletions(-) create mode 100644 be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp create mode 100644 be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h create mode 100644 be/src/format_v2/parquet/reader/native/common.cpp create mode 100644 be/src/format_v2/parquet/reader/native/common.h diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index f0b93e4e030a20..fd5c3775916bd1 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -69,6 +69,55 @@ static int num_children_node(const tparquet::SchemaElement& schema) { return schema.__isset.num_children ? schema.num_children : 0; } +static Status validate_native_schema_structure( + const std::vector& schemas) { + if (schemas.empty() || !is_group_node(schemas[0])) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + struct PendingGroup { + size_t remaining_children; + size_t depth; + }; + const auto root_children = num_children_node(schemas[0]); + if (root_children < 0 || static_cast(root_children) > schemas.size() - 1) { + return Status::InvalidArgument("Invalid parquet root child count {}", root_children); + } + std::vector pending {{static_cast(root_children), 0}}; + for (size_t pos = 1; pos < schemas.size(); ++pos) { + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (pending.empty()) { + return Status::InvalidArgument("Schema element {} is not reachable from the root", pos); + } + + const size_t depth = pending.back().depth + 1; + --pending.back().remaining_children; + const int children = num_children_node(schemas[pos]); + if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { + return Status::InvalidArgument("Invalid child count {} at schema element {}", children, + pos); + } + if (children > 0) { + // Bound the tree before any resize or recursion so an untrusted footer cannot + // turn a tiny schema into an unbounded allocation or parser stack. + if (depth > MAX_NATIVE_SCHEMA_DEPTH) { + return Status::InvalidArgument("Parquet schema depth {} exceeds limit {}", depth, + MAX_NATIVE_SCHEMA_DEPTH); + } + pending.push_back({static_cast(children), depth}); + } + } + while (!pending.empty() && pending.back().remaining_children == 0) { + pending.pop_back(); + } + if (!pending.empty()) { + return Status::InvalidArgument("Parquet schema ended before all children were parsed"); + } + return Status::OK(); +} + /** * `repeated_parent_def_level` is the definition level of the first ancestor node whose repetition_type equals REPEATED. * Empty array/map values are not stored in doris columns, so have to use `repeated_parent_def_level` to skip the @@ -134,9 +183,10 @@ std::string NativeFieldSchema::debug_string() const { Status NativeFieldDescriptor::parse_from_thrift( const std::vector& t_schemas) { - if (t_schemas.size() == 0 || !is_group_node(t_schemas[0])) { - return Status::InvalidArgument("Wrong parquet root schema element"); - } + _fields.clear(); + _physical_fields.clear(); + _name_to_field.clear(); + RETURN_IF_ERROR(validate_native_schema_structure(t_schemas)); const auto& root_schema = t_schemas[0]; _fields.resize(root_schema.num_children); _next_schema_pos = 1; diff --git a/be/src/format_v2/parquet/native_schema_desc.h b/be/src/format_v2/parquet/native_schema_desc.h index c74b3f9af02dc6..9ef01d062def99 100644 --- a/be/src/format_v2/parquet/native_schema_desc.h +++ b/be/src/format_v2/parquet/native_schema_desc.h @@ -37,6 +37,7 @@ namespace doris::format::parquet { // Constant for unassigned column IDs constexpr uint64_t NATIVE_UNASSIGNED_COLUMN_ID = UINT64_MAX; +constexpr size_t MAX_NATIVE_SCHEMA_DEPTH = 100; struct NativeFieldSchema { std::string name; @@ -150,6 +151,12 @@ class NativeFieldDescriptor { int32_t size() const { return cast_set(_fields.size()); } + size_t physical_fields_size() const { return _physical_fields.size(); } + + const NativeFieldSchema* get_physical_field(size_t index) const { + return _physical_fields[index]; + } + const std::vector& get_fields_schema() const { return _fields; } /** diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 03bb38daf095a9..daadbb1925f8e9 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -69,6 +69,33 @@ Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, // Native readers address projected leaves by stable DFS IDs. Assign them only on the private // v2 schema object so v1's cached schema lifecycle and numbering remain untouched. _schema.assign_ids(); + for (size_t row_group_idx = 0; row_group_idx < _metadata.row_groups.size(); ++row_group_idx) { + const auto& row_group = _metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.columns.size() != _schema.physical_fields_size()) { + // All v2 planners index chunks by the native DFS leaf order, so validate cardinality + // once before any projection, prefetch, or decoder can perform indexed access. + return Status::Corruption( + "Parquet row group {} has {} column chunks but schema has {} physical fields", + row_group_idx, row_group.columns.size(), _schema.physical_fields_size()); + } + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + if (chunk.meta_data.type != _schema.get_physical_field(column_idx)->physical_type) { + return Status::Corruption( + "Parquet row group {} column {} physical type {} does not match schema {}", + row_group_idx, column_idx, tparquet::to_string(chunk.meta_data.type), + tparquet::to_string(_schema.get_physical_field(column_idx)->physical_type)); + } + } + } return Status::OK(); } diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 67e9cd636604c5..971c7233053eb0 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -36,6 +36,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" #include "format_v2/parquet/reader/global_rowid_column_reader.h" +#include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/parquet/reader/row_position_column_reader.h" #include "util/defer_op.h" @@ -272,10 +273,17 @@ void materialize_count_star_placeholders(const format::FileScanRequest& request, } } -std::vector build_row_group_prefetch_ranges( +} // namespace + +namespace detail { + +Status build_native_prefetch_ranges( const tparquet::FileMetaData& metadata, const std::vector>& file_schema, - const std::vector& scan_columns, int row_group_idx) { + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges) { + DORIS_CHECK(ranges != nullptr); + ranges->clear(); std::unordered_set leaf_column_ids; for (const auto& projection : scan_columns) { const auto local_id = projection.local_id(); @@ -283,41 +291,56 @@ std::vector build_row_group_prefetch_ranges( local_id == format::GLOBAL_ROWID_COLUMN_ID) { continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); + if (local_id < 0 || local_id >= static_cast(file_schema.size()) || + file_schema[local_id] == nullptr) { + return Status::Corruption("Invalid Parquet projected column id {}", local_id); + } // Prefetch and merge-reader ranges must be physical leaf chunks, not Doris logical slots. // Example: for a struct column s, projecting only s.a should include only // the Parquet leaf chunk of a. Projecting the whole struct includes both a and b. collect_projected_leaf_column_ids(*file_schema[local_id], projection, &leaf_column_ids); } - DORIS_CHECK(row_group_idx >= 0 && row_group_idx < static_cast(metadata.row_groups.size())); + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + return Status::Corruption("Invalid Parquet row group index {}", row_group_idx); + } const auto& row_group_metadata = metadata.row_groups[row_group_idx]; std::vector ordered_leaf_column_ids(leaf_column_ids.begin(), leaf_column_ids.end()); std::ranges::sort(ordered_leaf_column_ids); - std::vector ranges; - ranges.reserve(ordered_leaf_column_ids.size()); + ranges->reserve(ordered_leaf_column_ids.size()); for (const auto leaf_column_id : ordered_leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && - leaf_column_id < static_cast(row_group_metadata.columns.size())); + if (leaf_column_id < 0 || + leaf_column_id >= static_cast(row_group_metadata.columns.size())) { + return Status::Corruption("Invalid Parquet leaf column id {}", leaf_column_id); + } const auto& chunk = row_group_metadata.columns[leaf_column_id]; if (!chunk.__isset.meta_data) { - continue; - } - const auto& column_metadata = chunk.meta_data; - const int64_t offset = column_metadata.__isset.dictionary_page_offset - ? column_metadata.dictionary_page_offset - : column_metadata.data_page_offset; - const int64_t size = column_metadata.total_compressed_size; - DORIS_CHECK(offset >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); + return Status::Corruption("Parquet leaf column {} has no chunk metadata", + leaf_column_id); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range(chunk.meta_data, file_size, + parquet_816_padding, &chunk_range)); + if (chunk_range.length > 0) { + if (chunk_range.offset > static_cast(std::numeric_limits::max()) || + chunk_range.length > static_cast(std::numeric_limits::max())) { + return Status::Corruption("Parquet column chunk range exceeds int64 coordinates"); + } + // Prefetch must use the same checked chunk extent as the decoder, including the + // PARQUET-816 compatibility padding, so warm-up cannot target different bytes. + ranges->push_back( + ParquetPageCacheRange {.offset = static_cast(chunk_range.offset), + .size = static_cast(chunk_range.length)}); } } - return ranges; + return Status::OK(); } +} // namespace detail + +namespace { + Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, const ParquetScanRange& scan_range, std::vector* row_group_first_rows, @@ -463,43 +486,33 @@ Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, return Status::OK(); } -namespace { - -int64_t native_column_start_offset(const tparquet::ColumnMetaData& column_metadata) { - return column_metadata.__isset.dictionary_page_offset ? column_metadata.dictionary_page_offset - : column_metadata.data_page_offset; -} - -bool is_native_row_group_outside_range(const tparquet::RowGroup& row_group, - const ParquetScanRange& scan_range) { - if (scan_range.size < 0) { - return false; - } - const int64_t range_start = scan_range.start_offset; - const int64_t range_end = range_start + scan_range.size; - DORIS_CHECK(range_start >= 0 && range_end >= range_start); - if (range_start == 0 && (scan_range.file_size < 0 || range_end >= scan_range.file_size)) { - return false; - } - if (row_group.columns.empty() || !row_group.columns.front().__isset.meta_data || - !row_group.columns.back().__isset.meta_data) { - return false; - } - const auto& first = row_group.columns.front().meta_data; - const auto& last = row_group.columns.back().meta_data; - const int64_t group_start = native_column_start_offset(first); - const int64_t group_end = native_column_start_offset(last) + last.total_compressed_size; - const int64_t group_mid = group_start + (group_end - group_start) / 2; - return group_mid < range_start || group_mid >= range_end; -} +namespace detail { Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, const ParquetScanRange& scan_range, std::vector* row_group_first_rows, std::vector* selected_row_groups) { DORIS_CHECK(row_group_first_rows != nullptr && selected_row_groups != nullptr); + if (scan_range.start_offset < 0 || scan_range.size < -1 || + (scan_range.size >= 0 && + scan_range.start_offset > std::numeric_limits::max() - scan_range.size)) { + return Status::Corruption("Invalid Parquet scan range [{}, {})", scan_range.start_offset, + scan_range.size); + } + const uint64_t range_start = static_cast(scan_range.start_offset); + const uint64_t range_end = scan_range.size < 0 + ? std::numeric_limits::max() + : range_start + static_cast(scan_range.size); + const size_t file_size = scan_range.file_size < 0 ? std::numeric_limits::max() + : static_cast(scan_range.file_size); + const bool full_file_range = + scan_range.size < 0 || (range_start == 0 && scan_range.file_size >= 0 && + range_end >= static_cast(scan_range.file_size)); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); row_group_first_rows->assign(metadata.row_groups.size(), 0); selected_row_groups->clear(); + selected_row_groups->reserve(metadata.row_groups.size()); int64_t next_first_row = 0; for (size_t row_group_idx = 0; row_group_idx < metadata.row_groups.size(); ++row_group_idx) { (*row_group_first_rows)[row_group_idx] = next_first_row; @@ -508,14 +521,47 @@ Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& meta return Status::Corruption("Invalid negative row count in parquet row group {}", row_group_idx); } + if (row_group.num_rows > std::numeric_limits::max() - next_first_row) { + return Status::Corruption("Parquet row counts overflow at row group {}", row_group_idx); + } next_first_row += row_group.num_rows; - if (!is_native_row_group_outside_range(row_group, scan_range)) { + bool selected = full_file_range; + if (!full_file_range) { + if (row_group.columns.empty()) { + return Status::Corruption("Parquet row group {} has no column chunks", + row_group_idx); + } + size_t group_start = std::numeric_limits::max(); + size_t group_end = 0; + for (size_t column_idx = 0; column_idx < row_group.columns.size(); ++column_idx) { + const auto& chunk = row_group.columns[column_idx]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet row group {} column {} has no metadata", + row_group_idx, column_idx); + } + native::ColumnChunkRange chunk_range; + RETURN_IF_ERROR(native::compute_column_chunk_range( + chunk.meta_data, file_size, compat.parquet_816_padding, &chunk_range)); + group_start = std::min(group_start, chunk_range.offset); + group_end = std::max(group_end, chunk_range.offset + chunk_range.length); + } + // Checked chunk ranges make end >= start; this midpoint form cannot overflow even + // when footer offsets are close to the host coordinate limit. + const uint64_t group_mid = + static_cast(group_start) + (group_end - group_start) / 2; + selected = group_mid >= range_start && group_mid < range_end; + } + if (selected) { selected_row_groups->push_back(cast_set(row_group_idx)); } } return Status::OK(); } +} // namespace detail + +namespace { + Status build_native_row_group_read_plans( const NativeParquetMetadata& metadata, const std::vector>& file_schema, @@ -580,7 +626,7 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, plan->pruning_stats = {}; std::vector row_group_first_rows; std::vector scan_range_selected; - RETURN_IF_ERROR(select_native_row_groups_by_scan_range( + RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( metadata.to_thrift(), scan_range, &row_group_first_rows, &scan_range_selected)); std::vector metadata_selected; RETURN_IF_ERROR(select_row_groups_by_metadata( @@ -1055,9 +1101,13 @@ Status ParquetScanScheduler::open_next_row_group( // row-group-scoped MergeRangeFileReader policy as v1. Sharing one wrapper is important: a // separate merge reader per leaf would duplicate its 128MB scratch capacity and defeat lazy // materialization for wide schemas. - const auto native_ranges = - build_row_group_prefetch_ranges(file_context.native_metadata->to_thrift(), file_schema, - request_scan_columns(request), row_group_idx); + const auto& thrift_metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + thrift_metadata.__isset.created_by ? thrift_metadata.created_by : std::string {}); + std::vector native_ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + thrift_metadata, file_schema, request_scan_columns(request), row_group_idx, + file_context.native_file->size(), compat.parquet_816_padding, &native_ranges)); _current_merge_range_active = file_context.set_native_random_access_ranges( native_ranges, detail::average_prefetch_range_size(native_ranges), _profile, _merge_read_slice_size); @@ -1098,8 +1148,8 @@ Status ParquetScanScheduler::open_next_row_group( // changes row/column materialization order. if (!_current_merge_range_active) { const auto prefetch_columns = adaptive_predicate_prefetch_columns(request); - prefetch_current_row_group_columns(file_context, file_schema, prefetch_columns, - &_current_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns( + file_context, file_schema, prefetch_columns, &_current_predicate_prefetched)); } for (const auto& col : request.non_predicate_columns) { const auto local_id = col.local_id(); @@ -1136,9 +1186,9 @@ Status ParquetScanScheduler::open_next_row_group( // With no row-level filters there is no lazy-read decision to wait for, so start warming // output chunks immediately after their readers are created. Filtered scans still defer // this until at least one row survives the predicate phase. - prefetch_current_row_group_columns(file_context, file_schema, - physical_non_predicate_columns(request), - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } *has_row_group = true; return Status::OK(); @@ -1771,24 +1821,29 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return compact_predicate_columns_with_profile(); } -void ParquetScanScheduler::prefetch_current_row_group_columns( +Status ParquetScanScheduler::prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched) { DORIS_CHECK(prefetched != nullptr); if (_current_merge_range_active || *prefetched || scan_columns.empty() || _current_row_group_id < 0 || file_context.native_metadata == nullptr) { - return; + return Status::OK(); } *prefetched = true; // The scanner request separates predicate and non-predicate columns so Parquet can read // predicate columns first and lazily materialize the rest. Keep the same contract for // prefetch: callers decide which side to warm, and this helper only translates that selected // projection into physical column-chunk byte ranges for the current row group. - file_context.prefetch_ranges( - build_row_group_prefetch_ranges(file_context.native_metadata->to_thrift(), file_schema, - scan_columns, _current_row_group_id), - nullptr); + const auto& metadata = file_context.native_metadata->to_thrift(); + const auto compat = native::parquet_reader_compat( + metadata.__isset.created_by ? metadata.created_by : std::string {}); + std::vector ranges; + RETURN_IF_ERROR(detail::build_native_prefetch_ranges( + metadata, file_schema, scan_columns, _current_row_group_id, + file_context.native_file->size(), compat.parquet_816_padding, &ranges)); + file_context.prefetch_ranges(ranges, nullptr); + return Status::OK(); } Status ParquetScanScheduler::read_current_row_group_batch( @@ -1884,9 +1939,9 @@ Status ParquetScanScheduler::read_current_row_group_batch( // Do not prefetch lazy output columns until at least one row survives filtering. This is // the same decision point where the v2 reader switches from predicate-only reads to // materializing non-predicate columns, so fully filtered batches avoid unnecessary IO. - prefetch_current_row_group_columns(file_context, file_schema, - physical_non_predicate_columns(request), - &_current_non_predicate_prefetched); + RETURN_IF_ERROR(prefetch_current_row_group_columns(file_context, file_schema, + physical_non_predicate_columns(request), + &_current_non_predicate_prefetched)); } { diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index e855baa4dba454..714d34c81f6ac9 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -60,6 +60,8 @@ namespace doris::format::parquet { struct ParquetFileContext; struct ParquetColumnSchema; +struct ParquetPageCacheRange; +struct ParquetScanRange; class NativeParquetMetadata; namespace detail { @@ -82,6 +84,15 @@ std::vector adaptive_prefetch_prefix( const std::unordered_map& stats, double minimum_reach_probability); bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence); +Status build_native_prefetch_ranges( + const tparquet::FileMetaData& metadata, + const std::vector>& file_schema, + const std::vector& scan_columns, int row_group_idx, + size_t file_size, bool parquet_816_padding, std::vector* ranges); +Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, + const ParquetScanRange& scan_range, + std::vector* row_group_first_rows, + std::vector* selected_row_groups); } // namespace detail // ============================================================================ @@ -214,7 +225,7 @@ class ParquetScanScheduler { const format::FileScanRequest& request, int row_group_idx, const tparquet::RowGroup& row_group_metadata); - void prefetch_current_row_group_columns( + Status prefetch_current_row_group_columns( ParquetFileContext& file_context, const std::vector>& file_schema, const std::vector& scan_columns, bool* prefetched); diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index 9c8019d6b6df8e..13304f23767429 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -47,12 +47,13 @@ #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr_context.h" -#include "format/parquet/parquet_block_split_bloom_filter.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" #include "format_v2/parquet/reader/native_column_reader.h" #include "format_v2/timestamp_statistics.h" #include "runtime/runtime_profile.h" +#include "storage/index/bloom_filter/bloom_filter.h" #include "storage/index/zone_map/zone_map_index.h" #include "storage/index/zone_map/zonemap_eval_context.h" #include "util/thrift_util.h" @@ -60,6 +61,36 @@ namespace doris::format::parquet { +namespace detail { + +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size) { + if (offset < 0 || header_size == 0 || payload_size < segment_v2::BloomFilter::MINIMUM_BYTES || + payload_size > segment_v2::BloomFilter::MAXIMUM_BYTES || payload_size % 32 != 0) { + return Status::Corruption( + "Invalid Parquet Bloom filter layout: offset {}, header {}, payload {}", offset, + header_size, payload_size); + } + const uint64_t unsigned_offset = static_cast(offset); + const uint64_t total_size = static_cast(header_size) + payload_size; + if (unsigned_offset > file_size || total_size > file_size - unsigned_offset) { + return Status::Corruption("Parquet Bloom filter range exceeds file size {}", file_size); + } + if (declared_length >= 0) { + const uint64_t unsigned_declared_length = static_cast(declared_length); + if (unsigned_declared_length < total_size || + unsigned_declared_length > file_size - unsigned_offset) { + return Status::Corruption( + "Parquet Bloom filter requires {} bytes, metadata declares {}, file has {}", + total_size, declared_length, file_size - unsigned_offset); + } + } + return Status::OK(); +} + +} // namespace detail + namespace { bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, @@ -76,15 +107,25 @@ enum class ParquetRowGroupPruneReason { Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, const io::FileReaderSPtr& file, io::IOContext* io_ctx, - std::unique_ptr* result) { + std::unique_ptr* result) { if (result == nullptr || file == nullptr || !metadata.__isset.bloom_filter_offset) { return Status::NotSupported("Parquet Bloom filter is unavailable"); } constexpr size_t MAX_BLOOM_HEADER_BYTES = 64; - const size_t header_read_size = - metadata.__isset.bloom_filter_length && metadata.bloom_filter_length > 0 - ? std::min(metadata.bloom_filter_length, MAX_BLOOM_HEADER_BYTES) - : MAX_BLOOM_HEADER_BYTES; + if (metadata.bloom_filter_offset < 0 || + (metadata.__isset.bloom_filter_length && metadata.bloom_filter_length <= 0)) { + return Status::Corruption("Invalid Parquet Bloom filter offset or declared length"); + } + const uint64_t bloom_offset = static_cast(metadata.bloom_filter_offset); + if (bloom_offset >= file->size()) { + return Status::Corruption("Parquet Bloom filter offset exceeds file size {}", file->size()); + } + const size_t available = file->size() - bloom_offset; + const size_t declared_available = + metadata.__isset.bloom_filter_length + ? std::min(metadata.bloom_filter_length, available) + : available; + const size_t header_read_size = std::min(declared_available, MAX_BLOOM_HEADER_BYTES); std::vector header_buffer(header_read_size); size_t bytes_read = 0; RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset, @@ -98,13 +139,20 @@ Status read_native_bloom_filter(const tparquet::ColumnMetaData& metadata, return Status::NotSupported("Unsupported Parquet Bloom filter encoding"); } + // Validate the complete split-block layout before allocating or adding footer-controlled + // offsets; BloomFilter::init() otherwise receives a truncated or oversized backing buffer. + RETURN_IF_ERROR(detail::validate_native_bloom_filter_layout( + metadata.bloom_filter_offset, header_size, header.numBytes, + metadata.__isset.bloom_filter_length ? metadata.bloom_filter_length : -1, + file->size())); + std::vector data(cast_set(header.numBytes)); - RETURN_IF_ERROR(file->read_at(metadata.bloom_filter_offset + header_size, + RETURN_IF_ERROR(file->read_at(static_cast(metadata.bloom_filter_offset) + header_size, Slice(data.data(), data.size()), &bytes_read, io_ctx)); if (bytes_read != data.size()) { return Status::Corruption("Truncated Parquet Bloom filter payload"); } - auto bloom_filter = std::make_unique(); + auto bloom_filter = std::make_unique(); RETURN_IF_ERROR(bloom_filter->init(reinterpret_cast(data.data()), data.size(), segment_v2::HashStrategyPB::XX_HASH_64)); *result = std::move(bloom_filter); @@ -875,6 +923,12 @@ ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( return result; } + if (statistics->__isset.null_count && statistics->null_count > column_value_count) { + // An impossible null count makes all derived min/max and all-null flags untrustworthy; + // disable pruning instead of turning corrupt footer metadata into false negatives. + return result; + } + const bool has_null_count = statistics->__isset.null_count && statistics->null_count >= 0; const int64_t null_count = has_null_count ? statistics->null_count : 0; const bool has_not_null = has_null_count ? column_value_count > null_count : true; @@ -1357,7 +1411,7 @@ ParquetRowGroupPruneReason native_bloom_filter_prune_reason( if (!chunk.__isset.meta_data) { continue; } - std::unique_ptr bloom_filter; + std::unique_ptr bloom_filter; Status status; { int64_t timer_sink = 0; diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 184a51997696e9..bc9143af1bef09 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -56,6 +56,12 @@ namespace doris::format::parquet { struct ParquetColumnSchema; struct ParquetFileContext; +namespace detail { +Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, + int64_t payload_size, int64_t declared_length, + size_t file_size); +} // namespace detail + // ============================================================================ // ============================================================================ diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp new file mode 100644 index 00000000000000..3421ecdd296892 --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.cpp @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" + +#include + +#include "util/hash_util.hpp" + +namespace doris::format::parquet::native { + +namespace { +Status set_hash_strategy(segment_v2::HashStrategyPB strategy, + std::function* hash_func) { + if (strategy != segment_v2::HashStrategyPB::XX_HASH_64) { + return Status::InvalidArgument("Invalid Parquet Bloom filter hash strategy {}", strategy); + } + *hash_func = [](const void* buf, int64_t len, uint64_t, void* out) { + *reinterpret_cast(out) = + HashUtil::xxhash64_compat_with_seed(reinterpret_cast(buf), len, 0); + }; + return Status::OK(); +} +} // namespace + +Status BlockSplitBloomFilter::init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) { + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _num_bytes = filter_size; + _size = _num_bytes; + _data = new char[_size]; + memset(_data, 0, _size); + _has_null = nullptr; + _is_write = true; + segment_v2::g_write_bloom_filter_num << 1; + segment_v2::g_write_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +Status BlockSplitBloomFilter::init(const char* buf, size_t size, + segment_v2::HashStrategyPB strategy) { + if (buf == nullptr || size <= 1) { + return Status::InvalidArgument("Invalid Parquet Bloom filter buffer of size {}", size); + } + RETURN_IF_ERROR(set_hash_strategy(strategy, &_hash_func)); + _data = new char[size]; + memcpy(_data, buf, size); + _size = size; + _num_bytes = size; + _has_null = nullptr; + segment_v2::g_read_bloom_filter_num << 1; + segment_v2::g_read_bloom_filter_total_bytes << _size; + segment_v2::g_total_bloom_filter_total_bytes << _size; + return Status::OK(); +} + +void BlockSplitBloomFilter::add_bytes(const char* buf, size_t size) { + DCHECK(buf != nullptr); + add_hash(hash(buf, size)); +} + +bool BlockSplitBloomFilter::test_bytes(const char* buf, size_t size) const { + return test_hash(hash(buf, size)); +} + +void BlockSplitBloomFilter::set_has_null(bool has_null) { + DCHECK(!has_null) << "Parquet Bloom filters do not track nulls"; +} + +void BlockSplitBloomFilter::set_masks(uint32_t key, BlockMask* block_mask) { + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + block_mask->item[i] = uint32_t {1} << ((key * SALT[i]) >> 27); + } +} + +void BlockSplitBloomFilter::add_hash(uint64_t hash) { + DCHECK_GE(_num_bytes, BYTES_PER_BLOCK); + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + bitset[bucket_index * BITS_SET_PER_BLOCK + i] |= block_mask.item[i]; + } +} + +bool BlockSplitBloomFilter::test_hash(uint64_t hash) const { + const uint32_t bucket_index = + static_cast((hash >> 32) * (_num_bytes / BYTES_PER_BLOCK) >> 32); + const auto* bitset = reinterpret_cast(_data); + BlockMask block_mask; + set_masks(static_cast(hash), &block_mask); + for (int i = 0; i < BITS_SET_PER_BLOCK; ++i) { + if ((bitset[bucket_index * BITS_SET_PER_BLOCK + i] & block_mask.item[i]) == 0) { + return false; + } + } + return true; +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h new file mode 100644 index 00000000000000..38dc97712ac46c --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/block_split_bloom_filter.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "storage/index/bloom_filter/bloom_filter.h" + +namespace doris::format::parquet::native { + +// Keep the split-block implementation inside v2; the native scan path must not link through the +// legacy Parquet reader merely to evaluate footer Bloom filters. +class BlockSplitBloomFilter final : public segment_v2::BloomFilter { +public: + Status init(uint64_t filter_size, segment_v2::HashStrategyPB strategy) override; + Status init(const char* buf, size_t size, segment_v2::HashStrategyPB strategy) override; + void add_bytes(const char* buf, size_t size) override; + bool test_bytes(const char* buf, size_t size) const override; + void set_has_null(bool has_null) override; + bool has_null() const override { return false; } + void add_hash(uint64_t hash) override; + bool test_hash(uint64_t hash) const override; + +private: + static constexpr int BYTES_PER_BLOCK = 32; + static constexpr int BITS_SET_PER_BLOCK = 8; + static constexpr uint32_t SALT[BITS_SET_PER_BLOCK] = {0x47b6137bU, 0x44974d91U, 0x8824ad5bU, + 0xa2b7289dU, 0x705495c7U, 0x2df1424bU, + 0x9efc4947U, 0x5c6bfb31U}; + + struct BlockMask { + uint32_t item[BITS_SET_PER_BLOCK]; + }; + + static void set_masks(uint32_t key, BlockMask* block_mask); +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp index 97d7608e1ba477..06e4b239f3abb4 100644 --- a/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_plain_decoder.cpp @@ -24,7 +24,6 @@ #include "core/column/column_vector.h" #include "core/types.h" -#include "format/parquet/parquet_common.h" #include "util/bit_util.h" namespace doris::format::parquet::native { diff --git a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp index 3f245d88043963..d541550cef180f 100644 --- a/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/bool_rle_decoder.cpp @@ -26,7 +26,6 @@ #include "core/column/column_vector.h" #include "core/types.h" -#include "format/parquet/parquet_common.h" #include "util/coding.h" #include "util/slice.h" diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h index 04f511a02f0d88..435688f485b217 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -28,7 +28,6 @@ #include "core/data_type/data_type.h" #include "core/types.h" #include "format/format_common.h" -#include "format/parquet/parquet_common.h" #include "format_v2/parquet/reader/native/decoder.h" #include "util/bit_util.h" #include "util/coding.h" diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index a696af8ffddd36..5dcbf46ccf9dde 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -1187,6 +1187,16 @@ Status ColumnChunkReader::load_page_nested_rows( if (UNLIKELY(rep_level < 0)) { return Status::Corruption("Parquet repetition level stream ended unexpectedly"); } + if constexpr (!OFFSET_INDEX && IN_COLLECTION) { + // A continuation level is valid across later V1 pages only after this chunk has seen + // a row start; accepting it on the first sequential page invents an orphan parent row. + if (!_nested_row_started && rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + _nested_row_started = true; + } if (rep_level == 0) { // rep_level 0 indicates start of new row if (*result_rows == max_rows) { // this page contain max_rows, page no end. _current_row += max_rows; @@ -1231,6 +1241,14 @@ Status ColumnChunkReader::load_cross_page_nested_ro if (UNLIKELY(rep_level < 0)) { return Status::Corruption("Parquet repetition level stream ended unexpectedly"); } + if constexpr (!OFFSET_INDEX && IN_COLLECTION) { + if (!_nested_row_started && rep_level != 0) { + return Status::Corruption( + "First Parquet nested data page starts with repetition level {}", + rep_level); + } + _nested_row_started = true; + } if (rep_level == 0) { // rep_level 0 indicates start of new row *cross_page = false; _rep_level_rewind_one(); diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 8106070bae9050..e35e6b675c730c 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -30,8 +30,8 @@ #include "core/column/column_string.h" #include "core/data_type/data_type.h" #include "core/data_type_serde/parquet_decode_source.h" -#include "format/parquet/parquet_common.h" #include "format_v2/parquet/native_schema_desc.h" +#include "format_v2/parquet/reader/native/common.h" #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" @@ -335,6 +335,7 @@ class ColumnChunkReader { Slice _v2_def_levels; bool _dict_checked = false; bool _has_dict = false; + bool _nested_row_started = false; Decoder* _page_decoder = nullptr; tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; // Map: encoding -> Decoder diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 25b7ec3795734a..0abc128b9b9f6b 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include "common/cast_set.h" @@ -103,6 +104,146 @@ bool is_direct_binary_type(PrimitiveType type) { return is_string_type(type) || type == TYPE_VARBINARY; } +Status validate_decimal_physical_type(const NativeFieldSchema& field, int precision, int scale) { + if (precision <= 0 || scale < 0 || scale > precision) { + return Status::Corruption("Parquet decimal field {} has invalid precision {} and scale {}", + field.name, precision, scale); + } + switch (field.physical_type) { + case tparquet::Type::INT32: + if (precision <= 9) { + return Status::OK(); + } + break; + case tparquet::Type::INT64: + if (precision <= 18) { + return Status::OK(); + } + break; + case tparquet::Type::BYTE_ARRAY: + return Status::OK(); + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { + const int length = + field.parquet_schema.__isset.type_length ? field.parquet_schema.type_length : -1; + if (length > 0) { + const int64_t max_precision = (static_cast(length) * 8 - 1) * 30103 / 100000; + if (precision <= max_precision) { + return Status::OK(); + } + } + break; + } + default: + break; + } + return Status::Corruption("Parquet decimal field {} has incompatible physical type {}", + field.name, tparquet::to_string(field.physical_type)); +} + +Status validate_physical_annotation(const NativeFieldSchema& field) { + const auto& schema = field.parquet_schema; + if (field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + (!schema.__isset.type_length || schema.type_length <= 0)) { + return Status::Corruption("Parquet fixed-length field {} has invalid width {}", field.name, + schema.__isset.type_length ? schema.type_length : -1); + } + auto require = [&](bool valid, std::string_view annotation) -> Status { + if (valid) { + return Status::OK(); + } + return Status::Corruption("Parquet {} field {} is incompatible with physical type {}", + annotation, field.name, tparquet::to_string(field.physical_type)); + }; + + if (schema.__isset.logicalType) { + const auto& logical = schema.logicalType; + if (logical.__isset.STRING || logical.__isset.ENUM || logical.__isset.JSON || + logical.__isset.BSON) { + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "string"); + } + if (logical.__isset.DECIMAL) { + return validate_decimal_physical_type(field, logical.DECIMAL.precision, + logical.DECIMAL.scale); + } + if (logical.__isset.DATE) { + return require(field.physical_type == tparquet::Type::INT32, "date"); + } + if (logical.__isset.TIME) { + const auto unit = parquet_time_unit(logical.TIME.unit); + return require( + (unit == ParquetTimeUnit::MILLIS && + field.physical_type == tparquet::Type::INT32) || + ((unit == ParquetTimeUnit::MICROS || unit == ParquetTimeUnit::NANOS) && + field.physical_type == tparquet::Type::INT64), + "time"); + } + if (logical.__isset.TIMESTAMP) { + return require( + field.physical_type == tparquet::Type::INT64 && + parquet_time_unit(logical.TIMESTAMP.unit) != ParquetTimeUnit::UNKNOWN, + "timestamp"); + } + if (logical.__isset.INTEGER) { + const int width = logical.INTEGER.bitWidth; + return require(((width == 8 || width == 16 || width == 32) && + field.physical_type == tparquet::Type::INT32) || + (width == 64 && field.physical_type == tparquet::Type::INT64), + "integer"); + } + if (logical.__isset.UUID) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 16, + "UUID"); + } + if (logical.__isset.FLOAT16) { + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 2, + "FLOAT16"); + } + if (logical.__isset.UNKNOWN) { + return Status::OK(); + } + return Status::Corruption("Unsupported Parquet logical annotation on field {}", field.name); + } + + if (!schema.__isset.converted_type) { + return Status::OK(); + } + switch (schema.converted_type) { + case tparquet::ConvertedType::UTF8: + case tparquet::ConvertedType::ENUM: + case tparquet::ConvertedType::JSON: + case tparquet::ConvertedType::BSON: + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "converted string"); + case tparquet::ConvertedType::DECIMAL: + return validate_decimal_physical_type(field, + schema.__isset.precision ? schema.precision : -1, + schema.__isset.scale ? schema.scale : -1); + case tparquet::ConvertedType::DATE: + case tparquet::ConvertedType::TIME_MILLIS: + case tparquet::ConvertedType::UINT_8: + case tparquet::ConvertedType::UINT_16: + case tparquet::ConvertedType::UINT_32: + case tparquet::ConvertedType::INT_8: + case tparquet::ConvertedType::INT_16: + case tparquet::ConvertedType::INT_32: + return require(field.physical_type == tparquet::Type::INT32, "converted INT32"); + case tparquet::ConvertedType::TIME_MICROS: + case tparquet::ConvertedType::TIMESTAMP_MILLIS: + case tparquet::ConvertedType::TIMESTAMP_MICROS: + case tparquet::ConvertedType::UINT_64: + case tparquet::ConvertedType::INT_64: + return require(field.physical_type == tparquet::Type::INT64, "converted INT64"); + case tparquet::ConvertedType::INTERVAL: + return require(field.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY && + schema.type_length == 12, + "interval"); + default: + return Status::Corruption("Unsupported Parquet converted annotation {} on field {}", + tparquet::to_string(schema.converted_type), field.name); + } +} + IColumn::Filter* conversion_failure_map(const NativeFieldSchema& field, const DataTypePtr& target_type, bool strict_mode, IColumn::Filter* output_null_map, @@ -170,6 +311,9 @@ bool serde_can_materialize_directly(const DataTypePtr& source_type, Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, ParquetDecodeContext* context) { DORIS_CHECK(context != nullptr); + // Annotation validation belongs before decoder construction: accepting an impossible pair + // lets the logical SerDe reinterpret a differently sized physical value stream. + RETURN_IF_ERROR(validate_physical_annotation(field)); switch (field.physical_type) { case tparquet::Type::BOOLEAN: context->physical_type = ParquetPhysicalType::BOOLEAN; @@ -511,11 +655,21 @@ Status ColumnReader::create(io::FileReaderSPtr file, NativeFieldSchema* field, reader.reset(struct_reader.release()); } else { auto physical_index = field->physical_column_index; + if (physical_index < 0 || static_cast(physical_index) >= row_group.columns.size()) { + // Keep the leaf-to-chunk invariant checked at this consumer too because unit callers + // can construct a reader without going through NativeParquetMetadata::init_schema(). + return Status::Corruption("Parquet physical column index {} is out of range {}", + physical_index, row_group.columns.size()); + } const auto offset_it = col_offsets.find(physical_index); const tparquet::OffsetIndex* offset_index = offset_it != col_offsets.end() ? &offset_it->second : nullptr; const tparquet::ColumnChunk& chunk = row_group.columns[physical_index]; + if (!chunk.__isset.meta_data) { + return Status::Corruption("Parquet physical column {} has no chunk metadata", + physical_index); + } if (in_collection) { if (offset_index == nullptr) { auto scalar_reader = ScalarColumnReader::create_unique( diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index f088d4e7d44d98..63c3017f9e0d62 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -31,9 +31,9 @@ #include "core/data_type/data_type.h" #include "format/column_type_convert.h" #include "format/generic_reader.h" -#include "format/parquet/parquet_common.h" #include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" +#include "format_v2/parquet/reader/native/common.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader_writer_fwd.h" diff --git a/be/src/format_v2/parquet/reader/native/common.cpp b/be/src/format_v2/parquet/reader/native/common.cpp new file mode 100644 index 00000000000000..f048d2b878ebee --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.cpp @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/parquet/reader/native/common.h" + +#include + +#include "core/types.h" +#include "util/simd/bits.h" + +namespace doris::format::parquet::native { + +Status FilterMap::init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all) { + _filter_all = filter_all; + _filter_map_data = filter_map_data; + _filter_map_size = filter_map_size; + if (filter_all) { + _has_filter = true; + _filter_ratio = 1; + } else if (filter_map_data == nullptr) { + _has_filter = false; + _filter_ratio = 0; + } else { + const size_t filter_count = simd::count_zero_num( + reinterpret_cast(filter_map_data), filter_map_size); + if (filter_count == filter_map_size) { + _has_filter = true; + _filter_all = true; + _filter_ratio = 1; + } else if (filter_count > 0 && filter_map_size > 0) { + _has_filter = true; + _filter_ratio = static_cast(filter_count) / filter_map_size; + } else { + _has_filter = false; + _filter_ratio = 0; + } + } + return Status::OK(); +} + +bool FilterMap::can_filter_all(size_t remaining_num_values, size_t filter_map_index) { + if (!_has_filter) { + return false; + } + if (_filter_all) { + DCHECK_LE(remaining_num_values + filter_map_index, _filter_map_size); + return true; + } + if (remaining_num_values + filter_map_index > _filter_map_size) { + return false; + } + return simd::count_zero_num( + reinterpret_cast(_filter_map_data + filter_map_index), + remaining_num_values) == remaining_num_values; +} + +Status FilterMap::generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index) const { + if (!has_filter() || filter_all()) { + return Status::InternalError( + "Native FilterMap requires a partial filter: has_filter={}, filter_all={}", + has_filter(), filter_all()); + } + if (rep_levels.empty()) { + return Status::OK(); + } + + nested_filter_map_data.resize(rep_levels.size()); + size_t current_row = current_row_ptr != nullptr ? *current_row_ptr : 0; + for (size_t i = start_index; i < rep_levels.size(); ++i) { + if (i != start_index && rep_levels[i] == 0) { + ++current_row; + if (current_row >= _filter_map_size) { + return Status::InvalidArgument("Nested filter row {} exceeds filter map size {}", + current_row, _filter_map_size); + } + } + nested_filter_map_data[i] = _filter_map_data[current_row]; + } + if (current_row_ptr != nullptr) { + *current_row_ptr = current_row; + } + + auto new_filter = std::make_unique(); + RETURN_IF_ERROR( + new_filter->init(nested_filter_map_data.data(), nested_filter_map_data.size(), false)); + *nested_filter_map = std::move(new_filter); + return Status::OK(); +} + +Status ColumnSelectVector::init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices) { + _num_values = num_values; + _num_nulls = 0; + _read_index = 0; + size_t map_index = 0; + bool is_null = false; + _has_filter = filter_map->has_filter(); + + if (_has_filter) { + _data_map.resize(num_values); + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_NULL; + } + } else { + for (size_t i = 0; i < run_length; ++i) { + _data_map[map_index++] = FILTERED_CONTENT; + } + } + is_null = !is_null; + } + + size_t num_read = 0; + size_t source_index = 0; + size_t valid_count = 0; + while (valid_count < num_values) { + DCHECK_LT(filter_map_index + source_index, filter_map->filter_map_size()); + if (skipped_indices != nullptr && + skipped_indices->contains(filter_map_index + source_index)) { + ++source_index; + continue; + } + if (filter_map->filter_map_data()[filter_map_index + source_index] != 0) { + _data_map[valid_count] = + _data_map[valid_count] == FILTERED_NULL ? NULL_DATA : CONTENT; + ++num_read; + } + ++valid_count; + ++source_index; + } + _num_filtered = num_values - num_read; + + if (null_map != nullptr && num_read > 0) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_read); + if (_num_nulls == 0) { + memset(null_map->data() + null_map_index, 0, num_read); + } else if (_num_nulls == num_values) { + memset(null_map->data() + null_map_index, 1, num_read); + } else { + for (size_t i = 0; i < num_values; ++i) { + if (_data_map[i] == CONTENT) { + (*null_map)[null_map_index++] = static_cast(false); + } else if (_data_map[i] == NULL_DATA) { + (*null_map)[null_map_index++] = static_cast(true); + } + } + } + } + } else { + _num_filtered = 0; + _run_length_null_map = &run_length_null_map; + if (null_map != nullptr) { + size_t null_map_index = null_map->size(); + null_map->resize(null_map_index + num_values); + for (const auto run_length : run_length_null_map) { + memset(null_map->data() + null_map_index, is_null ? 1 : 0, run_length); + null_map_index += run_length; + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } else { + for (const auto run_length : run_length_null_map) { + if (is_null) { + _num_nulls += run_length; + } + is_null = !is_null; + } + } + } + return Status::OK(); +} + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/common.h b/be/src/format_v2/parquet/reader/native/common.h new file mode 100644 index 00000000000000..4ad204da2ff30d --- /dev/null +++ b/be/src/format_v2/parquet/reader/native/common.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/status.h" +#include "core/column/column_nullable.h" + +namespace doris::format::parquet::native { + +// These cursor types belong to the native v2 decoder so its execution path never reaches into +// the legacy Parquet reader for filtering or level materialization. +using level_t = int16_t; + +class FilterMap { +public: + Status init(const uint8_t* filter_map_data, size_t filter_map_size, bool filter_all); + + Status generate_nested_filter_map(const std::vector& rep_levels, + std::vector& nested_filter_map_data, + std::unique_ptr* nested_filter_map, + size_t* current_row_ptr, size_t start_index = 0) const; + + const uint8_t* filter_map_data() const { return _filter_map_data; } + size_t filter_map_size() const { return _filter_map_size; } + bool has_filter() const { return _has_filter; } + bool filter_all() const { return _filter_all; } + double filter_ratio() const { return _has_filter ? _filter_ratio : 0; } + + bool can_filter_all(size_t remaining_num_values, size_t filter_map_index); + +private: + bool _filter_all = false; + bool _has_filter = false; + const uint8_t* _filter_map_data = nullptr; + size_t _filter_map_size = 0; + double _filter_ratio = 0; +}; + +class ColumnSelectVector { +public: + enum DataReadType : uint8_t { CONTENT = 0, NULL_DATA, FILTERED_CONTENT, FILTERED_NULL }; + + Status init(const std::vector& run_length_null_map, size_t num_values, + NullMap* null_map, FilterMap* filter_map, size_t filter_map_index, + const std::unordered_set* skipped_indices = nullptr); + + size_t num_values() const { return _num_values; } + size_t num_nulls() const { return _num_nulls; } + size_t num_filtered() const { return _num_filtered; } + bool has_filter() const { return _has_filter; } + + template + size_t get_next_run(DataReadType* data_read_type) { + DCHECK_EQ(_has_filter, has_filter); + if constexpr (has_filter) { + if (_read_index == _num_values) { + return 0; + } + const DataReadType type = _data_map[_read_index++]; + size_t run_length = 1; + while (_read_index < _num_values && _data_map[_read_index] == type) { + ++run_length; + ++_read_index; + } + *data_read_type = type; + return run_length; + } + + size_t run_length = 0; + while (run_length == 0) { + if (_read_index == _run_length_null_map->size()) { + return 0; + } + *data_read_type = _read_index % 2 == 0 ? CONTENT : NULL_DATA; + run_length = (*_run_length_null_map)[_read_index++]; + } + return run_length; + } + +private: + std::vector _data_map; + const std::vector* _run_length_null_map = nullptr; + bool _has_filter = false; + size_t _num_values = 0; + size_t _num_nulls = 0; + size_t _num_filtered = 0; + size_t _read_index = 0; +}; + +} // namespace doris::format::parquet::native diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index dbc5fef2536cf2..9f12caaec4304f 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -36,7 +36,6 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type.h" #include "exec/common/arithmetic_overflow.h" -#include "format/parquet/parquet_common.h" #include "format_v2/parquet/reader/native/decoder.h" #include "util/bit_stream_utils.h" #include "util/bit_stream_utils.inline.h" diff --git a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h index fbe7d12c3522e2..c6c6784cd35f76 100644 --- a/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/fix_length_plain_decoder.h @@ -22,13 +22,8 @@ #include "common/status.h" #include "core/column/column_fixed_length_object.h" #include "core/data_type/data_type.h" -#include "format/parquet/parquet_common.h" #include "format_v2/parquet/reader/native/decoder.h" -namespace doris { -class ColumnSelectVector; -} // namespace doris - namespace doris::format::parquet::native { class FixLengthPlainDecoder final : public Decoder { diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.cpp b/be/src/format_v2/parquet/reader/native/level_decoder.cpp index 52113c951b82c4..618d6ebf9f6d45 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.cpp +++ b/be/src/format_v2/parquet/reader/native/level_decoder.cpp @@ -23,7 +23,6 @@ #include #include "common/cast_set.h" -#include "format/parquet/parquet_common.h" #include "util/bit_stream_utils.inline.h" #include "util/bit_util.h" #include "util/coding.h" diff --git a/be/src/format_v2/parquet/reader/native/level_decoder.h b/be/src/format_v2/parquet/reader/native/level_decoder.h index e232fe3a7be022..9dcff491eb0759 100644 --- a/be/src/format_v2/parquet/reader/native/level_decoder.h +++ b/be/src/format_v2/parquet/reader/native/level_decoder.h @@ -24,7 +24,7 @@ #include #include "common/status.h" -#include "format/parquet/parquet_common.h" +#include "format_v2/parquet/reader/native/common.h" #include "util/bit_stream_utils.h" #include "util/rle_encoding.h" #include "util/slice.h" diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp index 2e503d4f31fe15..285717e64ea714 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -26,7 +26,6 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/config.h" -#include "format/parquet/parquet_common.h" #include "io/fs/buffered_reader.h" #include "runtime/runtime_profile.h" #include "storage/cache/page_cache.h" diff --git a/be/src/format_v2/parquet/reader/native/page_reader.h b/be/src/format_v2/parquet/reader/native/page_reader.h index a798a01f9f4a53..56103a8994d059 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.h +++ b/be/src/format_v2/parquet/reader/native/page_reader.h @@ -26,7 +26,6 @@ #include "common/cast_set.h" #include "common/config.h" #include "common/status.h" -#include "format/parquet/parquet_common.h" #include "storage/cache/page_cache.h" #include "util/block_compression.h" namespace doris { diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 487bb747c9ab49..4f06ebb75e3358 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -288,7 +288,7 @@ Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_ return Status::OK(); } - ::doris::FilterMap filter; + native::FilterMap filter; RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); _native_reader->reset_filter_map_index(); ColumnPtr native_column(std::move(column)); diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index ad287ed8cec308..22531645608306 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -28,7 +28,6 @@ #include #include -#include "format/parquet/parquet_common.h" #include "format_v2/column_data.h" #include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/reader/column_reader.h" diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index ab99f94989a1b7..493af2b29c81c6 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -390,6 +390,79 @@ TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { } } +TEST(ParquetV2NativeDecoderTest, InvalidLogicalPhysicalPairsFailBeforeDecode) { + auto invalid = [](tparquet::Type::type physical, const tparquet::LogicalType& logical) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + + tparquet::LogicalType date; + date.__set_DATE(tparquet::DateType()); + EXPECT_FALSE(invalid(tparquet::Type::BOOLEAN, date).ok()); + + tparquet::LogicalType timestamp; + timestamp.__set_TIMESTAMP(tparquet::TimestampType()); + timestamp.TIMESTAMP.__set_unit(tparquet::TimeUnit()); + timestamp.TIMESTAMP.unit.__set_MICROS(tparquet::MicroSeconds()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, timestamp).ok()); + + tparquet::LogicalType integer; + integer.__set_INTEGER(tparquet::IntType()); + integer.INTEGER.__set_bitWidth(64); + integer.INTEGER.__set_isSigned(true); + EXPECT_FALSE(invalid(tparquet::Type::INT32, integer).ok()); + + tparquet::LogicalType string; + string.__set_STRING(tparquet::StringType()); + EXPECT_FALSE(invalid(tparquet::Type::INT32, string).ok()); + + tparquet::LogicalType uuid; + uuid.__set_UUID(tparquet::UUIDType()); + EXPECT_FALSE(invalid(tparquet::Type::BYTE_ARRAY, uuid).ok()); + + auto invalid_converted = [](tparquet::Type::type physical, + tparquet::ConvertedType::type converted, int type_length = -1, + int precision = -1, int scale = -1) { + NativeFieldSchema field; + field.name = "bad"; + field.physical_type = physical; + field.parquet_schema.__set_converted_type(converted); + if (type_length >= 0) { + field.parquet_schema.__set_type_length(type_length); + } + if (precision >= 0) { + field.parquet_schema.__set_precision(precision); + } + if (scale >= 0) { + field.parquet_schema.__set_scale(scale); + } + ParquetDecodeContext context; + return init_decode_context_for_test(field, nullptr, &context); + }; + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UTF8).ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::TIME_MICROS).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::UINT_64).ok()); + EXPECT_FALSE(invalid_converted(tparquet::Type::FIXED_LEN_BYTE_ARRAY, + tparquet::ConvertedType::INTERVAL, 8) + .ok()); + EXPECT_FALSE( + invalid_converted(tparquet::Type::INT32, tparquet::ConvertedType::DECIMAL, -1, 10, 2) + .ok()); + + NativeFieldSchema fixed; + fixed.name = "fixed"; + fixed.physical_type = tparquet::Type::FIXED_LEN_BYTE_ARRAY; + ParquetDecodeContext context; + EXPECT_FALSE(init_decode_context_for_test(fixed, nullptr, &context).ok()); + fixed.parquet_schema.__set_type_length(4); + EXPECT_TRUE(init_decode_context_for_test(fixed, nullptr, &context).ok()); +} + TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) { DataTypePtr datetime_type = make_nullable(std::make_shared(6)); NativeFieldSchema int96_field; @@ -1389,6 +1462,63 @@ TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) .is()); } +TEST(ParquetV2NativeDecoderTest, FirstNestedV1PageRejectsOrphanContinuation) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + // A two-value RLE run at repetition level 1 has no level-0 row start. + const std::vector payload {2, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0}; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(2); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + EXPECT_TRUE(load_malformed_nested_page(header, payload, 2).is()); +} + +TEST(ParquetV2NativeDecoderTest, NestedV1ContinuationRemainsValidAfterFirstRowStart) { + auto make_page = [](int values, std::vector payload) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + return serialize_page(header, payload); + }; + auto first_page = make_page(2, {2, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0}); + auto second_page = make_page(1, {2, 0, 0, 0, 2, 1, 0, 0, 0, 0}); + first_page.insert(first_page.end(), second_page.begin(), second_page.end()); + + MemoryBufferedReader reader(first_page); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(3); + chunk.meta_data.__set_total_compressed_size(first_page.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + field.repetition_level = 1; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + std::vector levels; + size_t rows = 0; + bool cross_page = false; + ASSERT_TRUE(chunk_reader.load_page_nested_rows(levels, 1, &rows, &cross_page).ok()); + ASSERT_TRUE(cross_page); + ASSERT_TRUE(chunk_reader.load_cross_page_nested_row(levels, &cross_page).ok()); + EXPECT_FALSE(cross_page); + EXPECT_EQ(levels, std::vector({0, 1, 1})); +} + TEST(ParquetV2NativeDecoderTest, HugeNestedPageCountsDoNotPreallocateFromHeaders) { for (auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { tparquet::PageHeader header; diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 7028832a66dc2e..8ab62ccb8275d7 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -81,6 +81,64 @@ const ColumnString& string_data_column(const IColumn& column) { return assert_cast(column); } +TEST(ParquetScanMetadataSafetyTest, CheckedChunkRangesDrivePrefetchAndSplitAssignment) { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + metadata.__set_schema({root, leaf}); + + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + column.__set_data_page_offset(100); + column.__set_dictionary_page_offset(-1); + column.__set_total_compressed_size(20); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + row_group.__set_columns({chunk}); + metadata.__set_row_groups({row_group}); + + auto schema = std::make_unique(); + schema->local_id = 0; + schema->leaf_column_id = 0; + schema->type = std::make_shared(); + std::vector> file_schema; + file_schema.push_back(std::move(schema)); + std::vector ranges; + ASSERT_TRUE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); + ASSERT_EQ(ranges.size(), 1); + EXPECT_EQ(ranges[0].offset, 100); + EXPECT_EQ(ranges[0].size, 20); + + std::vector first_rows; + std::vector selected; + format::parquet::ParquetScanRange first_split { + .start_offset = 0, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, first_split, &first_rows, &selected) + .ok()); + EXPECT_TRUE(selected.empty()); + format::parquet::ParquetScanRange second_split { + .start_offset = 100, .size = 100, .file_size = 200}; + ASSERT_TRUE(format::parquet::detail::select_native_row_groups_by_scan_range( + metadata, second_split, &first_rows, &selected) + .ok()); + EXPECT_EQ(selected, std::vector({0})); + + metadata.row_groups[0].columns[0].meta_data.__set_data_page_offset(190); + EXPECT_FALSE(format::parquet::detail::build_native_prefetch_ranges( + metadata, file_schema, {field_projection(0)}, 0, 200, false, &ranges) + .ok()); +} + class Int32ZoneMapExpr final : public VExpr { public: enum class Op { GE, GT, LT }; diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 9c87572866cb8f..8ef1ad1143abf9 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include @@ -30,6 +31,7 @@ #include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" namespace doris::format::parquet { namespace { @@ -626,4 +628,82 @@ TEST(ParquetSchemaTest, NativeGroupEnumLogicalTypeIsRejected) { std::string::npos); } +TEST(ParquetSchemaTest, NativeSchemaRejectsUnboundedChildCountsBeforeAllocation) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(std::numeric_limits::max()); + NativeFieldDescriptor descriptor; + EXPECT_FALSE(descriptor.parse_from_thrift({root}).ok()); + + root.__set_num_children(1); + tparquet::SchemaElement nested; + nested.__set_name("nested"); + nested.__set_num_children(std::numeric_limits::max()); + nested.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + EXPECT_FALSE(descriptor.parse_from_thrift({root, nested}).ok()); +} + +std::vector nested_native_schema(size_t depth) { + std::vector schema; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + schema.push_back(root); + for (size_t level = 0; level < depth; ++level) { + tparquet::SchemaElement group; + group.__set_name("g" + std::to_string(level)); + group.__set_num_children(1); + group.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(group); + } + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + schema.push_back(leaf); + return schema; +} + +TEST(ParquetSchemaTest, NativeSchemaBoundsRecursiveDepth) { + NativeFieldDescriptor accepted; + EXPECT_TRUE(accepted.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH)).ok()); + NativeFieldDescriptor rejected; + EXPECT_FALSE( + rejected.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH + 1)).ok()); +} + +TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingMetadata) { + auto make_metadata = []() { + tparquet::FileMetaData metadata; + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(2); + tparquet::SchemaElement first; + first.__set_name("a"); + first.__set_type(tparquet::Type::INT32); + first.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::SchemaElement second = first; + second.__set_name("b"); + metadata.__set_schema({root, first, second}); + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + return metadata; + }; + + { + NativeParquetMetadata metadata(make_metadata(), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } + { + auto thrift = make_metadata(); + thrift.row_groups[0].columns.resize(2); + tparquet::ColumnMetaData first_meta; + first_meta.__set_type(tparquet::Type::INT32); + thrift.row_groups[0].columns[0].__set_meta_data(first_meta); + NativeParquetMetadata metadata(std::move(thrift), 0); + EXPECT_FALSE(metadata.init_schema(true, false).ok()); + } +} + } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index 0788f09b736cfc..77a43b02493d19 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -569,6 +569,33 @@ TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossib EXPECT_EQ(statistics.max_value.get(), 3); } +TEST(ParquetStatisticsTransformTest, InvalidFooterNullCountDisablesPruningStatistics) { + auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), + {int32_array({7})}); + auto reader = make_reader(table, 1, false, true); + auto schema = build_file_schema(*reader); + tparquet::Statistics malformed; + malformed.__set_null_count(2); + malformed.__set_min_value(encoded_value(7)); + malformed.__set_max_value(encoded_value(7)); + const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( + *schema[0], &malformed, 1); + EXPECT_FALSE(statistics.has_null_count); + EXPECT_FALSE(statistics.has_min_max); +} + +TEST(ParquetBloomFilterPruningTest, NativeBloomLayoutRejectsSubBlockAndHugePayloads) { + using format::parquet::detail::validate_native_bloom_filter_layout; + EXPECT_TRUE(validate_native_bloom_filter_layout(8, 12, 32, 44, 128).ok()); + EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, 2, 14, 128).ok()); + EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, 33, 45, 128).ok()); + EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, std::numeric_limits::max(), -1, + std::numeric_limits::max()) + .ok()); + EXPECT_FALSE(validate_native_bloom_filter_layout(120, 12, 32, 44, 128).ok()); + EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, 32, 200, 128).ok()); +} + TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), arrow::field("d", arrow::float64(), false)}), diff --git a/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out b/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out index 5441e21cdf8dcb..1a2c37fe9dfe7a 100644 --- a/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out +++ b/regression-test/data/nereids_rules_p0/pkfk/eliminate_inner.out @@ -144,7 +144,7 @@ PhysicalResultSink --NestedLoopJoin[INNER_JOIN] ----filter((pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(fkt_not_null.f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) +----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) ------PhysicalOlapScan[fkt_not_null] -- !res -- @@ -155,9 +155,9 @@ pk with filter that not same as fk -- !shape -- PhysicalResultSink --NestedLoopJoin[INNER_JOIN] -----filter((cast(pkt.p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) +----filter((cast(p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(fkt_not_null.f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) +----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt_not_null.fk = 1)) ------PhysicalOlapScan[fkt_not_null] -- !res -- @@ -167,7 +167,7 @@ simple_case -- !shape -- PhysicalResultSink ---filter(( not fkt.fk IS NULL)) +--filter(( not fk IS NULL)) ----PhysicalOlapScan[fkt] -- !res -- @@ -300,7 +300,7 @@ PhysicalResultSink --NestedLoopJoin[INNER_JOIN] ----filter((pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(fkt.f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) +----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) ------PhysicalOlapScan[fkt] -- !res -- @@ -311,9 +311,9 @@ pk with filter that not same as fk -- !shape -- PhysicalResultSink --NestedLoopJoin[INNER_JOIN] -----filter((cast(pkt.p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) +----filter((cast(p as DECIMALV3(38, 6)) = 1.000000) and (pkt.pk = 1)) ------PhysicalOlapScan[pkt] -----filter((cast(fkt.f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) +----filter((cast(f as DECIMALV3(38, 6)) = 1.000000) and (fkt.fk = 1)) ------PhysicalOlapScan[fkt] -- !res -- @@ -325,7 +325,7 @@ multi_table_join_with_pk_predicate PhysicalResultSink --hashJoin[INNER_JOIN] hashCondition=((fkt_not_null.fk = fkt_not_null2.fk)) otherCondition=() ----hashJoin[INNER_JOIN] hashCondition=((pkt.pk = fkt_not_null.fk)) otherCondition=() -------filter((cast(pkt.p as DECIMALV3(38, 6)) = 1.000000)) +------filter((cast(p as DECIMALV3(38, 6)) = 1.000000)) --------PhysicalOlapScan[pkt] ------PhysicalOlapScan[fkt_not_null] ----PhysicalOlapScan[fkt_not_null(fkt_not_null2)] From dbd48eb72dcae4272aff57f0e0dbaef4fba0a2f9 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 07:10:31 +0800 Subject: [PATCH 17/34] [fix](be) address native parquet review and CI failures --- .../expr/equality_delete_predicate.cpp | 61 +++++++++-- .../format_v2/parquet/native_schema_desc.cpp | 45 ++++++-- .../parquet/parquet_file_context.cpp | 13 +-- .../format_v2/parquet/parquet_file_context.h | 4 +- be/src/format_v2/parquet/parquet_reader.cpp | 103 ++---------------- be/src/format_v2/parquet/parquet_reader.h | 3 +- be/src/format_v2/parquet/parquet_scan.cpp | 21 ++++ be/src/format_v2/parquet/parquet_scan.h | 1 + .../format_v2/parquet/parquet_statistics.cpp | 38 ++++++- be/src/format_v2/parquet/parquet_statistics.h | 3 + .../format_v2/parquet/reader/column_reader.h | 1 + .../parquet/reader/native/column_reader.cpp | 12 +- .../parquet/reader/native/column_reader.h | 8 +- .../reader/native/delta_bit_pack_decoder.h | 23 +++- .../parquet/reader/native/page_reader.cpp | 38 ++++++- .../parquet/reader/native_column_reader.cpp | 55 +++++----- .../parquet/reader/native_column_reader.h | 7 +- be/src/format_v2/table/iceberg_reader.cpp | 5 +- be/src/format_v2/table_reader.cpp | 8 +- .../format_v2/parquet/native_decoder_test.cpp | 35 +++++- .../parquet/parquet_reader_control_test.cpp | 27 +++++ .../format_v2/parquet/parquet_reader_test.cpp | 57 +++++++++- .../format_v2/parquet/parquet_schema_test.cpp | 101 +++++++++++++++++ .../parquet/parquet_statistics_test.cpp | 30 +++++ 24 files changed, 523 insertions(+), 176 deletions(-) diff --git a/be/src/format_v2/expr/equality_delete_predicate.cpp b/be/src/format_v2/expr/equality_delete_predicate.cpp index 4d5c511abdb181..1d111aa7a74930 100644 --- a/be/src/format_v2/expr/equality_delete_predicate.cpp +++ b/be/src/format_v2/expr/equality_delete_predicate.cpp @@ -26,28 +26,63 @@ #include "core/assert_cast.h" #include "core/block/column_with_type_and_name.h" #include "core/column/column_nullable.h" +#include "core/column/column_string.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "util/hash_util.hpp" namespace doris::format { namespace { bool column_value_equal(const ColumnPtr& lhs, size_t lhs_row, const ColumnPtr& rhs, size_t rhs_row) { - if (lhs->is_nullable() && rhs->is_nullable()) { - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; - } + const IColumn* lhs_data = lhs.get(); + const IColumn* rhs_data = rhs.get(); if (lhs->is_nullable()) { const auto& nullable_lhs = assert_cast(*lhs); - return !nullable_lhs.is_null_at(lhs_row) && - nullable_lhs.get_nested_column().compare_at(lhs_row, rhs_row, *rhs, -1) == 0; + if (nullable_lhs.is_null_at(lhs_row)) { + return rhs->is_nullable() && + assert_cast(*rhs).is_null_at(rhs_row); + } + lhs_data = &nullable_lhs.get_nested_column(); } if (rhs->is_nullable()) { const auto& nullable_rhs = assert_cast(*rhs); - return !nullable_rhs.is_null_at(rhs_row) && - lhs->compare_at(lhs_row, rhs_row, nullable_rhs.get_nested_column(), -1) == 0; + if (nullable_rhs.is_null_at(rhs_row)) { + return false; + } + rhs_data = &nullable_rhs.get_nested_column(); + } + const bool lhs_binary = check_and_get_column(*lhs_data) != nullptr || + check_and_get_column(*lhs_data) != nullptr; + const bool rhs_binary = check_and_get_column(*rhs_data) != nullptr || + check_and_get_column(*rhs_data) != nullptr; + if (lhs_binary && rhs_binary) { + // Iceberg schema evolution may represent the same byte key as STRING in one file and + // VARBINARY in another. Equality-delete semantics compare bytes, not column storage classes. + return lhs_data->get_data_at(lhs_row) == rhs_data->get_data_at(rhs_row); + } + return lhs_data->compare_at(lhs_row, rhs_row, *rhs_data, -1) == 0; +} + +void update_varbinary_hashes(const ColumnWithTypeAndName& entry, uint64_t* hashes) { + const IColumn* data = entry.column.get(); + const uint8_t* null_map = nullptr; + if (entry.column->is_nullable()) { + const auto& nullable = assert_cast(*entry.column); + data = &nullable.get_nested_column(); + null_map = nullable.get_null_map_data().data(); + } + for (size_t row = 0; row < entry.column->size(); ++row) { + if (null_map != nullptr && null_map[row] != 0) { + hashes[row] = HashUtil::xxHash64NullWithSeed(hashes[row]); + continue; + } + const auto bytes = data->get_data_at(row); + hashes[row] = HashUtil::xxHash64WithSeed(bytes.data, bytes.size, hashes[row]); } - return lhs->compare_at(lhs_row, rhs_row, *rhs, -1) == 0; } } // namespace @@ -155,8 +190,14 @@ ColumnPtr EqualityDeletePredicate::_evaluate_key_block(const Block& data_key_blo std::vector EqualityDeletePredicate::_build_hashes(const Block& block) { std::vector hashes(block.rows(), 0); - for (const auto& column : block.get_columns()) { - column->update_hashes_with_value(hashes.data(), nullptr); + for (const auto& entry : block) { + if (remove_nullable(entry.type)->get_primitive_type() == TYPE_VARBINARY) { + // ColumnVarbinary intentionally lacks the generic column hash hook. Keep the V2 delete + // hash byte-identical to ColumnString so schema-mapped binary keys share hash buckets. + update_varbinary_hashes(entry, hashes.data()); + } else { + entry.column->update_hashes_with_value(hashes.data(), nullptr); + } } return hashes; } diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index fd5c3775916bd1..cc9ba4b403cb84 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -71,7 +71,16 @@ static int num_children_node(const tparquet::SchemaElement& schema) { static Status validate_native_schema_structure( const std::vector& schemas) { - if (schemas.empty() || !is_group_node(schemas[0])) { + if (schemas.empty()) { + return Status::InvalidArgument("Wrong parquet root schema element"); + } + + const auto& root = schemas[0]; + if (root.__isset.type || !root.__isset.num_children || root.num_children <= 0 || + (root.__isset.repetition_type && + root.repetition_type != tparquet::FieldRepetitionType::REQUIRED)) { + // Writers may encode the root's implicit REQUIRED repetition explicitly, but an optional + // or repeated root would make every descendant's definition/repetition levels ambiguous. return Status::InvalidArgument("Wrong parquet root schema element"); } @@ -94,6 +103,19 @@ static Status validate_native_schema_structure( const size_t depth = pending.back().depth + 1; --pending.back().remaining_children; + const auto& schema = schemas[pos]; + if (!schema.__isset.repetition_type) { + return Status::InvalidArgument("Schema element {} has no repetition type", pos); + } + if (schema.__isset.type == schema.__isset.num_children) { + // Every non-root node is exactly one of primitive or group. Rejecting both missing and + // dual kind fields prevents the parser from guessing based on a default child count. + return Status::InvalidArgument("Schema element {} has ambiguous primitive/group kind", + pos); + } + if (schema.__isset.num_children && schema.num_children <= 0) { + return Status::InvalidArgument("Group schema element {} has no children", pos); + } const int children = num_children_node(schemas[pos]); if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { return Status::InvalidArgument("Invalid child count {} at schema element {}", children, @@ -326,7 +348,8 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( tparquet::LogicalType logicalType, bool nullable) { std::pair ans = {std::make_shared(), false}; bool& is_type_compatibility = ans.second; - if (logicalType.__isset.STRING) { + if (logicalType.__isset.STRING || logicalType.__isset.ENUM || logicalType.__isset.JSON || + logicalType.__isset.BSON) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.DECIMAL) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_DECIMAL128I, nullable, @@ -358,7 +381,10 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( } } } else if (logicalType.__isset.TIME) { - ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); + const int scale = logicalType.TIME.unit.__isset.MILLIS ? 3 : 6; + // TIME stores an integer unit, so its Doris scale must preserve the footer unit or + // sub-second values are silently truncated by the target SerDe. + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, scale); } else if (logicalType.__isset.TIMESTAMP) { if (_enable_mapping_timestamp_tz) { if (logicalType.TIMESTAMP.isAdjustedToUTC) { @@ -371,8 +397,6 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( } ans.first = DataTypeFactory::instance().create_data_type( TYPE_DATETIMEV2, nullable, 0, logicalType.TIMESTAMP.unit.__isset.MILLIS ? 3 : 6); - } else if (logicalType.__isset.JSON) { - ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); } else if (logicalType.__isset.UUID) { if (_enable_mapping_varbinary) { ans.first = DataTypeFactory::instance().create_data_type(TYPE_VARBINARY, nullable, -1, @@ -394,6 +418,9 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( bool& is_type_compatibility = ans.second; switch (physical_schema.converted_type) { case tparquet::ConvertedType::type::UTF8: + case tparquet::ConvertedType::type::ENUM: + case tparquet::ConvertedType::type::JSON: + case tparquet::ConvertedType::type::BSON: ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); break; case tparquet::ConvertedType::type::DECIMAL: @@ -404,9 +431,10 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATEV2, nullable); break; case tparquet::ConvertedType::type::TIME_MILLIS: - [[fallthrough]]; + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 3); + break; case tparquet::ConvertedType::type::TIME_MICROS: - ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable); + ans.first = DataTypeFactory::instance().create_data_type(TYPE_TIMEV2, nullable, 0, 6); break; case tparquet::ConvertedType::type::TIMESTAMP_MILLIS: ans.first = DataTypeFactory::instance().create_data_type(TYPE_DATETIMEV2, nullable, 0, 3); @@ -439,9 +467,6 @@ std::pair NativeFieldDescriptor::convert_to_doris_type( is_type_compatibility = true; ans.first = DataTypeFactory::instance().create_data_type(TYPE_LARGEINT, nullable); break; - case tparquet::ConvertedType::type::JSON: - ans.first = DataTypeFactory::instance().create_data_type(TYPE_STRING, nullable); - break; default: throw Exception(Status::InternalError("Not supported parquet ConvertedType: {}", physical_schema.converted_type)); diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index daadbb1925f8e9..1f804d6ba6b8a2 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -713,7 +713,7 @@ Status arrow_status_to_doris_status(const arrow::Status& status) { Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, - bool enable_mapping_timestamp_tz) { + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { DORIS_CHECK(input_file_reader != nullptr); if (detail::should_stage_small_http_file(input_file_reader->path().native(), input_file_reader->size(), @@ -732,7 +732,8 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); auto meta_cache_key = FileMetaCache::get_key(native_file, file_description); meta_cache_key.append("\0v2", 3); - meta_cache_key.push_back(static_cast(true)); + // Schema mapping is part of the cached value, so both flags must participate in its identity. + meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); size_t native_footer_size = 0; if (meta_cache != nullptr && meta_cache->enabled() && @@ -742,7 +743,7 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont } else { RETURN_IF_ERROR(parse_native_parquet_footer( native_file, &native_metadata_owner, &native_footer_size, io_ctx, - /*enable_mapping_varbinary=*/true, enable_mapping_timestamp_tz)); + enable_mapping_varbinary, enable_mapping_timestamp_tz)); ++native_footer_read_calls; if (meta_cache != nullptr && meta_cache->enabled()) { meta_cache->insert(meta_cache_key, native_metadata_owner.release(), @@ -974,12 +975,6 @@ Status ParquetFileContext::load_native_page_indexes( return Status::OK(); } -void ParquetFileContext::register_page_cache_ranges(std::vector ranges) { - // Native column readers register exact page payloads themselves; retaining a second range map - // would recreate the removed Arrow metadata adapter's cache path. - (void)ranges; -} - void ParquetFileContext::prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx) { io::FileReaderSPtr reader = native_file; diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index 46854179b2915c..ebac63b3e75993 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -197,7 +197,7 @@ struct ParquetFileContext { Status open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); Status load_native_offset_indexes( int row_group_id, const std::unordered_set& leaf_column_ids, std::unordered_map* offset_indexes) const; @@ -206,8 +206,6 @@ struct ParquetFileContext { std::unordered_map* page_indexes, int64_t* read_time = nullptr, int64_t* parse_time = nullptr) const; - // Retained as a compatibility hook for callers; native readers admit exact page payloads. - void register_page_cache_ranges(std::vector ranges); // Best-effort asynchronous warm-up for Parquet column chunks. This only has an effect when // the underlying Doris file reader is a CachedRemoteFileReader; other readers keep the same // random-access behavior and simply skip prefetch. diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 99dbfa1bd5b294..c0c2630e1532f2 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -63,63 +63,6 @@ int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_m : cast_set(column_metadata.data_page_offset()); } -void collect_all_leaf_column_ids(const ParquetColumnSchema& column_schema, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (column_schema.leaf_column_id >= 0) { - leaf_column_ids->insert(column_schema.leaf_column_id); - } - return; - } - for (const auto& child : column_schema.children) { - DORIS_CHECK(child != nullptr); - collect_all_leaf_column_ids(*child, leaf_column_ids); - } -} - -void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex& projection, - std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - if (projection.project_all_children || projection.children.empty()) { - collect_all_leaf_column_ids(column_schema, leaf_column_ids); - return; - } - for (const auto& child_projection : projection.children) { - const auto child_it = - std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { - return child_schema->local_id == child_projection.local_id(); - }); - DORIS_CHECK(child_it != column_schema.children.end()); - collect_projected_leaf_column_ids(**child_it, child_projection, leaf_column_ids); - } -} - -void collect_request_leaf_column_ids( - const std::vector>& file_schema, - const format::FileScanRequest& request, std::unordered_set* leaf_column_ids) { - DORIS_CHECK(leaf_column_ids != nullptr); - auto collect_scan_column = [&](const format::LocalColumnIndex& projection) { - const auto local_id = projection.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID || - local_id == format::GLOBAL_ROWID_COLUMN_ID) { - return; - } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); - collect_projected_leaf_column_ids(*file_schema[local_id], projection, leaf_column_ids); - }; - for (const auto& column : request.predicate_columns) { - collect_scan_column(column); - } - for (const auto& column : request.non_predicate_columns) { - if (!request.is_count_star_placeholder(column.column_id())) { - collect_scan_column(column); - } - } -} - Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { if (!column_schema.type_descriptor.unsupported_reason.empty()) { @@ -176,34 +119,6 @@ Status validate_requested_columns_supported( return Status::OK(); } -std::vector build_page_cache_ranges( - const tparquet::FileMetaData& metadata, - const std::vector>& file_schema, - const format::FileScanRequest& request, const RowGroupScanPlan& row_group_plan) { - std::unordered_set leaf_column_ids; - collect_request_leaf_column_ids(file_schema, request, &leaf_column_ids); - std::vector ranges; - ranges.reserve(row_group_plan.row_groups.size() * leaf_column_ids.size()); - for (const auto& row_group_plan_item : row_group_plan.row_groups) { - const auto& row_group_metadata = metadata.row_groups[row_group_plan_item.row_group_id]; - for (const auto leaf_column_id : leaf_column_ids) { - DORIS_CHECK(leaf_column_id >= 0 && - leaf_column_id < static_cast(row_group_metadata.columns.size())); - const auto& column_metadata = row_group_metadata.columns[leaf_column_id].meta_data; - const int64_t offset = column_metadata.__isset.dictionary_page_offset - ? column_metadata.dictionary_page_offset - : column_metadata.data_page_offset; - const int64_t size = column_metadata.total_compressed_size; - DORIS_CHECK(offset >= 0); - DORIS_CHECK(size >= 0); - if (size > 0) { - ranges.push_back(ParquetPageCacheRange {.offset = offset, .size = size}); - } - } - } - return ranges; -} - const ParquetColumnSchema& projected_root_schema( const std::vector>& file_schema, const format::LocalColumnIndex& projection) { @@ -385,10 +300,11 @@ ParquetReader::ParquetReader(std::shared_ptr& system_p std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context, - bool enable_mapping_timestamp_tz) + bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) : FileReader(system_properties, file_description, io_ctx, profile), _global_rowid_context(global_rowid_context), - _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz) {} + _enable_mapping_timestamp_tz(enable_mapping_timestamp_tz), + _enable_mapping_varbinary(enable_mapping_varbinary) {} ParquetReader::~ParquetReader() = default; @@ -427,9 +343,9 @@ Status ParquetReader::init(RuntimeState* state) { // around the whole operation so footer/cache latency cannot disappear from a slow profile. { SCOPED_TIMER(_parquet_profile.parse_footer_time); - RETURN_IF_ERROR(_state->file_context.open(_tracing_file_reader, _io_ctx.get(), - _state->enable_page_cache, *_file_description, - _enable_mapping_timestamp_tz)); + RETURN_IF_ERROR(_state->file_context.open( + _tracing_file_reader, _io_ctx.get(), _state->enable_page_cache, *_file_description, + _enable_mapping_timestamp_tz, _enable_mapping_varbinary)); } if (_profile != nullptr) { COUNTER_UPDATE(_parquet_profile.file_footer_read_calls, @@ -548,11 +464,8 @@ Status ParquetReader::open(std::shared_ptr request) { if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } - if (_state->enable_page_cache) { - _state->file_context.register_page_cache_ranges( - build_page_cache_ranges(_state->file_context.native_metadata->to_thrift(), - _state->file_schema, *request_snapshot, row_group_plan)); - } + // Native page readers admit exact validated page payloads to cache. Do not pre-register whole + // column chunks here: footer offsets are untrusted and this obsolete range map is not consumed. _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); _state->scheduler.set_global_rowid_context(_global_rowid_context); diff --git a/be/src/format_v2/parquet/parquet_reader.h b/be/src/format_v2/parquet/parquet_reader.h index 6c8e88cc27a9b6..da6135b81ae838 100644 --- a/be/src/format_v2/parquet/parquet_reader.h +++ b/be/src/format_v2/parquet/parquet_reader.h @@ -46,7 +46,7 @@ class ParquetReader : public format::FileReader { std::unique_ptr& file_description, std::shared_ptr io_ctx, RuntimeProfile* profile, std::optional global_rowid_context = std::nullopt, - bool enable_mapping_timestamp_tz = false); + bool enable_mapping_timestamp_tz = false, bool enable_mapping_varbinary = false); ~ParquetReader() override; Status init(RuntimeState* state) override; @@ -89,6 +89,7 @@ class ParquetReader : public format::FileReader { std::optional _global_rowid_context; // global RowId context size_t _batch_size = ParquetScanScheduler::DEFAULT_READ_BATCH_SIZE; bool _enable_mapping_timestamp_tz = false; // whether UTC timestamps are mapped to TIMESTAMPTZ + bool _enable_mapping_varbinary = false; // whether raw BYTE_ARRAY is mapped to VARBINARY }; } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 971c7233053eb0..b16af939ae2234 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -165,6 +165,11 @@ bool supports_row_level_dictionary_filter(const ParquetColumnSchema& column_sche column_metadata.type != tparquet::Type::BYTE_ARRAY) { return false; } + if (remove_nullable(column_schema.type)->get_primitive_type() == TYPE_VARBINARY) { + // A table STRING predicate can be rewritten through a raw VARBINARY file slot. Evaluating + // it on dictionary Fields before the mapping expression is neither type-safe nor exact. + return false; + } // Row-level dictionary filtering consumes dictionary ids from DATA_PAGE payloads. It is exact // only when every data page is dictionary encoded. Mixed dictionary/plain chunks are left on // the normal decoded-value path, matching the safety rule used by StarRocks and Doris v1. @@ -990,6 +995,18 @@ void ParquetScanScheduler::flush_current_reader_profiles() { } } +bool ParquetScanScheduler::finish_current_reader_batch_profiles() { + bool crossed_page = false; + // A scheduler batch is counted once even when several projected leaves cross page boundaries. + for (const auto& reader : _current_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + for (const auto& reader : _current_non_predicate_columns | std::views::values) { + crossed_page |= reader->crossed_page_since_last_batch(); + } + return crossed_page; +} + const detail::PredicateConjunctSchedule& ParquetScanScheduler::predicate_conjunct_schedule( const format::FileScanRequest& request) { if (_predicate_schedule_request == &request) { @@ -1855,6 +1872,10 @@ Status ParquetScanScheduler::read_current_row_group_batch( // every tiny batch is measurable on wide/nested scans, so flush periodically and force the // tail at row-group reset/close. Defer profile_flush {[this, batch_rows]() { + if (finish_current_reader_batch_profiles() && + _scan_profile.column_reader_profile.page_crossing_batches != nullptr) { + COUNTER_UPDATE(_scan_profile.column_reader_profile.page_crossing_batches, 1); + } const bool finishes_row_group = _current_range_idx + 1 == _current_selected_ranges.size() && _current_range_rows_read + batch_rows == _current_selected_ranges[_current_range_idx].length; diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 714d34c81f6ac9..253f6d1696960c 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -202,6 +202,7 @@ class ParquetScanScheduler { void reset_current_row_group(); void flush_current_reader_profiles(); + bool finish_current_reader_batch_profiles(); const detail::PredicateConjunctSchedule& predicate_conjunct_schedule( const format::FileScanRequest& request); std::vector adaptive_predicate_prefetch_columns( diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index 13304f23767429..214e7988bd1768 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -89,6 +89,23 @@ Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, return Status::OK(); } +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics) { + const bool binary = type_descriptor.physical_type == ::parquet::Type::BYTE_ARRAY || + type_descriptor.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + if (!binary) { + return true; + } + if (statistics.__isset.min_value || statistics.__isset.max_value) { + // Do not combine a type-defined bound with a deprecated bound: the two fields can use + // different byte ordering, so a mixed pair cannot form one valid interval. + return statistics.__isset.min_value && statistics.__isset.max_value; + } + // Deprecated binary min/max fields were ordered with signed bytes by legacy parquet-mr, while + // Parquet's type-defined order is unsigned. Only an equal pair is independent of that mismatch. + return statistics.__isset.min && statistics.__isset.max && statistics.min == statistics.max; +} + } // namespace detail namespace { @@ -1234,6 +1251,13 @@ Status select_row_groups_by_metadata( namespace { +bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.type != nullptr); + // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in + // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. + return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; +} + bool check_native_statistics(const tparquet::RowGroup& row_group, const std::vector>& file_schema, const format::FileScanRequest& request, @@ -1250,6 +1274,7 @@ bool check_native_statistics(const tparquet::RowGroup& row_group, } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } @@ -1259,6 +1284,10 @@ bool check_native_statistics(const tparquet::RowGroup& row_group, const auto& column_metadata = chunk.meta_data; const auto* statistics = column_metadata.__isset.statistics ? &column_metadata.statistics : nullptr; + if (statistics != nullptr && !detail::can_use_native_footer_min_max( + column_schema->type_descriptor, *statistics)) { + statistics = nullptr; + } zone_map = ParquetStatisticsUtils::MakeZoneMap( ParquetStatisticsUtils::TransformColumnStatistics( *column_schema, statistics, column_metadata.num_values, timezone)); @@ -1346,6 +1375,11 @@ ParquetRowGroupPruneReason native_dictionary_prune_reason( column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } + if (!native_metadata_predicate_is_type_safe(*column_schema)) { + // The file-local VARBINARY may feed a table-side STRING cast. Pruning before that cast + // can compare different Field kinds and incorrectly discard a matching row group. + continue; + } const auto& chunk = row_group.columns[column_schema->leaf_column_id]; if (!chunk.__isset.meta_data || (chunk.meta_data.type != tparquet::Type::BYTE_ARRAY && @@ -1403,6 +1437,7 @@ ParquetRowGroupPruneReason native_bloom_filter_prune_reason( } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema) || !bloom_filter_supported(*column_schema) || column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; @@ -2224,7 +2259,8 @@ Status select_row_group_ranges_by_native_page_index( continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr) { + if (column_schema == nullptr || column_schema->type == nullptr || + !native_metadata_predicate_is_type_safe(*column_schema)) { continue; } const auto index_it = page_indexes.find(column_schema->leaf_column_id); diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index bc9143af1bef09..aa1f3b614f0d72 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -55,11 +55,14 @@ namespace doris::format::parquet { struct ParquetColumnSchema; struct ParquetFileContext; +struct ParquetTypeDescriptor; namespace detail { Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, int64_t payload_size, int64_t declared_length, size_t file_size); +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics); } // namespace detail // ============================================================================ diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index ccffa2fa85e291..b060c7a4ca71bd 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -63,6 +63,7 @@ class ParquetColumnReader { // Native statistics are cumulative and can be recursively aggregated for complex columns. // Flush once at the scheduler batch boundary instead of snapshotting after each operation. virtual void flush_profile() {} + virtual bool crossed_page_since_last_batch() { return false; } virtual Result dictionary_values() { return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 0abc128b9b9f6b..11689dd5fb3104 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -1171,11 +1171,14 @@ Status ScalarColumnReader::read_column_levels(Filte template Result ScalarColumnReader::convert_dict_column_to_string_column( - const ColumnInt32* dict_column) { + const ColumnInt32* dict_column, const DataTypePtr& target_type) { DORIS_CHECK(dict_column != nullptr); + DORIS_CHECK(target_type != nullptr); Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); DORIS_CHECK(dictionary_decoder != nullptr); - const DataTypePtr dictionary_type = remove_nullable(_field_schema->data_type); + // A Hive STRING can be backed by an unannotated Parquet BYTE_ARRAY that native metadata maps to + // VARBINARY. The dictionary must use the projected table type, exactly like normal data pages. + const DataTypePtr dictionary_type = remove_nullable(target_type); const DataTypeSerDeSPtr dictionary_serde = dictionary_type->get_serde(); if (_materialization_state.dictionary_generation != dictionary_decoder->dictionary_generation()) { @@ -1213,7 +1216,8 @@ ScalarColumnReader::convert_dict_column_to_string_c } template -Result ScalarColumnReader::dictionary_values() { +Result ScalarColumnReader::dictionary_values( + const DataTypePtr& target_type) { Decoder* dictionary_decoder = _chunk_reader->dictionary_decoder(); if (dictionary_decoder == nullptr || dictionary_decoder->dictionary_size() == 0) { return ResultError(Status::NotSupported("Parquet column has no reusable dictionary")); @@ -1226,7 +1230,7 @@ Result ScalarColumnReader::dictio } // Materialize the typed dictionary once and keep it in _materialization_state. Later row-level // filtering decodes only ids and flattens surviving values from this same dictionary. - return convert_dict_column_to_string_column(ids.get()); + return convert_dict_column_to_string_column(ids.get(), target_type); } template diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 63c3017f9e0d62..22683a70e5ac6f 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -196,11 +196,11 @@ class ColumnReader { bool* eof) = 0; virtual Result convert_dict_column_to_string_column( - const ColumnInt32* dict_column) { + const ColumnInt32* dict_column, const DataTypePtr& target_type) { throw Exception( Status::FatalError("Method convert_dict_column_to_string_column is not supported")); } - virtual Result dictionary_values() { + virtual Result dictionary_values(const DataTypePtr& target_type) { return ResultError(Status::NotSupported("Parquet dictionary values are not supported")); } @@ -270,8 +270,8 @@ class ScalarColumnReader : public ColumnReader { Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof) override; Result convert_dict_column_to_string_column( - const ColumnInt32* dict_column) override; - Result dictionary_values() override; + const ColumnInt32* dict_column, const DataTypePtr& target_type) override; + Result dictionary_values(const DataTypePtr& target_type) override; const std::vector& get_rep_level() const override { return _rep_levels; } const std::vector& get_def_level() const override { return _def_levels; } ColumnStatistics column_statistics() override { diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index 9f12caaec4304f..b3abc55b18d03f 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -42,6 +42,21 @@ #include "util/slice.h" namespace doris::format::parquet::native { +namespace detail { + +inline Status checked_delta_padding_bits(uint32_t bit_width, uint32_t remaining_values, + int64_t* padding_bits) { + DORIS_CHECK(padding_bits != nullptr); + const uint64_t widened_bits = static_cast(bit_width) * remaining_values; + if (UNLIKELY(widened_bits > static_cast(std::numeric_limits::max()))) { + return Status::Corruption("Parquet delta miniblock padding is too large"); + } + *padding_bits = static_cast(widened_bits); + return Status::OK(); +} + +} // namespace detail + class DeltaDecoder : public Decoder { public: DeltaDecoder() = default; @@ -657,7 +672,13 @@ Status DeltaBitPackDecoder::_get_internal(T* buffer, uint32_t num_values, _total_values_remaining -= num_values; if (_total_values_remaining == 0) [[unlikely]] { - if (!_bit_reader->Advance(_delta_bit_width * _values_remaining_current_mini_block)) { + int64_t padding_bits = 0; + // Footer-controlled miniblock counts can exceed 32-bit products. Widen before multiplying; + // BitReader::Advance then verifies that the full declared padding exists in the stream. + RETURN_IF_ERROR(detail::checked_delta_padding_bits(cast_set(_delta_bit_width), + _values_remaining_current_mini_block, + &padding_bits)); + if (!_bit_reader->Advance(padding_bits)) { return Status::IOError("Skip padding EOF"); } _values_remaining_current_mini_block = 0; diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp index 285717e64ea714..b679ed25ab9feb 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -53,7 +53,35 @@ Status PageReader::_validate_page_header(uint32_t h // Sizes are untrusted signed Thrift fields and must fit the column chunk before arithmetic. return Status::Corruption("Parquet page payload exceeds its column chunk"); } - if (_cur_page_header.__isset.data_page_header_v2) { + + const bool has_v1 = _cur_page_header.__isset.data_page_header; + const bool has_v2 = _cur_page_header.__isset.data_page_header_v2; + const bool has_dictionary = _cur_page_header.__isset.dictionary_page_header; + const bool has_index = _cur_page_header.__isset.index_page_header; + bool matching_layout = false; + switch (_cur_page_header.type) { + case tparquet::PageType::DATA_PAGE: + matching_layout = has_v1 && !has_v2 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DATA_PAGE_V2: + matching_layout = has_v2 && !has_v1 && !has_dictionary && !has_index; + break; + case tparquet::PageType::DICTIONARY_PAGE: + matching_layout = has_dictionary && !has_v1 && !has_v2 && !has_index; + break; + case tparquet::PageType::INDEX_PAGE: + matching_layout = has_index && !has_v1 && !has_v2 && !has_dictionary; + break; + default: + break; + } + if (UNLIKELY(!matching_layout)) { + // The type discriminant owns the only valid union member. Guessing from an optional header + // lets a malformed page use one layout for validation and another for decoding. + return Status::Corruption("Parquet page type does not match its page-header layout"); + } + + if (_cur_page_header.type == tparquet::PageType::DATA_PAGE_V2) { const auto& v2 = _cur_page_header.data_page_header_v2; if (UNLIKELY(v2.num_values < 0 || v2.num_rows < 0 || v2.num_nulls < 0 || v2.repetition_levels_byte_length < 0 || @@ -71,10 +99,10 @@ Status PageReader::_validate_page_header(uint32_t h static_cast(_cur_page_header.uncompressed_page_size))) { return Status::Corruption("Parquet data page v2 level bytes exceed the page payload"); } - } else if (_cur_page_header.__isset.data_page_header && + } else if (_cur_page_header.type == tparquet::PageType::DATA_PAGE && UNLIKELY(_cur_page_header.data_page_header.num_values < 0)) { return Status::Corruption("Parquet data page has a negative value count"); - } else if (_cur_page_header.__isset.dictionary_page_header && + } else if (_cur_page_header.type == tparquet::PageType::DICTIONARY_PAGE && UNLIKELY(_cur_page_header.dictionary_page_header.num_values < 0)) { return Status::Corruption("Parquet dictionary page has a negative value count"); } @@ -146,6 +174,8 @@ Status PageReader::parse_page_header() { Slice s = _page_cache_handle.data(); real_header_size = cast_set(s.size); SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Thrift does not clear absent optional fields when reusing an output object. + _cur_page_header = tparquet::PageHeader {}; auto st = deserialize_thrift_msg(reinterpret_cast(s.data), &real_header_size, true, &_cur_page_header); if (!st.ok()) return st; @@ -205,6 +235,8 @@ Status PageReader::parse_page_header() { } real_header_size = cast_set(header_size); SCOPED_RAW_TIMER(&_page_statistics.decode_header_time); + // Reset on every retry as a partial deserialize can otherwise leak a stale union member. + _cur_page_header = tparquet::PageHeader {}; auto st = deserialize_thrift_msg(page_header_buf, &real_header_size, true, &_cur_page_header); if (st.ok()) { diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 4f06ebb75e3358..c191ee50e09b90 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -519,8 +519,8 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& const auto* matched_id_column = check_and_get_column(*_matched_dictionary_ids); DORIS_CHECK(matched_id_column != nullptr); - auto string_values = - DORIS_TRY(_native_reader->convert_dict_column_to_string_column(matched_id_column)); + auto string_values = DORIS_TRY( + _native_reader->convert_dict_column_to_string_column(matched_id_column, _type)); RETURN_IF_ERROR(append_non_null_dictionary_values(column, std::move(string_values))); if (_profile.reader_select_rows != nullptr) { COUNTER_UPDATE(_profile.reader_select_rows, selected_rows); @@ -531,27 +531,42 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& } void NativeColumnReader::flush_profile() { - int64_t max_leaf_page_reads = 0; - record_page_fragments(sync_native_profile(&max_leaf_page_reads), max_leaf_page_reads); + record_page_fragments(sync_native_profile()); +} + +bool NativeColumnReader::crossed_page_since_last_batch() { + if (_native_reader == nullptr) { + return false; + } + const auto stats = _native_reader->column_statistics(); + bool crossed_page = false; + if (stats.leaf_page_read_counters.size() == _batch_leaf_page_read_counters.size()) { + for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { + crossed_page |= + stats.leaf_page_read_counters[leaf] - _batch_leaf_page_read_counters[leaf] > 1; + } + } else { + // The first snapshot covers the first batch; later snapshots must keep the stable tree shape. + for (const int64_t page_reads : stats.leaf_page_read_counters) { + crossed_page |= page_reads > 1; + } + } + _batch_leaf_page_read_counters = stats.leaf_page_read_counters; + return crossed_page; } Result NativeColumnReader::dictionary_values() { DORIS_CHECK(_native_reader != nullptr); - return _native_reader->dictionary_values(); + return _native_reader->dictionary_values(_type); } -void NativeColumnReader::record_page_fragments(int64_t page_fragments, - int64_t max_leaf_page_reads) { +void NativeColumnReader::record_page_fragments(int64_t page_fragments) { if (_profile.native_page_fragments != nullptr) { COUNTER_UPDATE(_profile.native_page_fragments, page_fragments); } - // Summed fragments from MAP/STRUCT siblings do not mean any individual leaf crossed a page. - if (max_leaf_page_reads > 1 && _profile.page_crossing_batches != nullptr) { - COUNTER_UPDATE(_profile.page_crossing_batches, 1); - } } -int64_t NativeColumnReader::sync_native_profile(int64_t* max_leaf_page_reads) { +int64_t NativeColumnReader::sync_native_profile() { if (_native_reader == nullptr) { return 0; } @@ -620,22 +635,6 @@ int64_t NativeColumnReader::sync_native_profile(int64_t* max_leaf_page_reads) { stats.read_page_header_time - reported.read_page_header_time); } const int64_t page_read_delta = stats.page_read_counter - reported.page_read_counter; - int64_t max_leaf_delta = 0; - if (stats.leaf_page_read_counters.size() == reported.leaf_page_read_counters.size()) { - for (size_t leaf = 0; leaf < stats.leaf_page_read_counters.size(); ++leaf) { - max_leaf_delta = - std::max(max_leaf_delta, stats.leaf_page_read_counters[leaf] - - reported.leaf_page_read_counters[leaf]); - } - } else { - // The tree shape is stable after initialization; retain a safe first-snapshot fallback. - for (const int64_t page_reads : stats.leaf_page_read_counters) { - max_leaf_delta = std::max(max_leaf_delta, page_reads); - } - } - if (max_leaf_page_reads != nullptr) { - *max_leaf_page_reads = max_leaf_delta; - } if (_profile.page_read_count != nullptr) { COUNTER_UPDATE(_profile.page_read_count, page_read_delta); } diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 22531645608306..5bfe0ccc15aa4a 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -89,6 +89,7 @@ class NativeColumnReader final : public ParquetColumnReader { MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter) override; void flush_profile() override; + bool crossed_page_since_last_batch() override; Result dictionary_values() override; private: @@ -107,8 +108,8 @@ class NativeColumnReader final : public ParquetColumnReader { Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, MutableColumnPtr& column, const DataTypePtr& output_type, bool dictionary_ids, int64_t* rows_read); - int64_t sync_native_profile(int64_t* max_leaf_page_reads = nullptr); - void record_page_fragments(int64_t page_fragments, int64_t max_leaf_page_reads); + int64_t sync_native_profile(); + void record_page_fragments(int64_t page_fragments); Status validate_selected_span(int64_t rows); void advance_selected_span(int64_t rows); @@ -130,6 +131,8 @@ class NativeColumnReader final : public ParquetColumnReader { // The native tree exposes cumulative statistics. Keep the last reported snapshot so each // FileScannerV2 batch contributes only its delta to RuntimeProfile. native::ColumnReader::ColumnStatistics _reported_native_stats; + // Page-crossing is sampled at every scheduler batch, independently of amortized profile flushes. + std::vector _batch_leaf_page_read_counters; std::vector _filter_scratch; size_t _batches_since_scratch_check = 0; MutableColumnPtr _skip_column; diff --git a/be/src/format_v2/table/iceberg_reader.cpp b/be/src/format_v2/table/iceberg_reader.cpp index d770cc095404b3..1c6f1935e015a8 100644 --- a/be/src/format_v2/table/iceberg_reader.cpp +++ b/be/src/format_v2/table/iceberg_reader.cpp @@ -651,10 +651,13 @@ Status IcebergTableReader::_create_delete_file_reader(const TIcebergDeleteFileDe std::shared_ptr io_ctx(&delete_io_ctx->io_ctx, [](io::IOContext*) {}); const bool enable_mapping_timestamp_tz = scan_params.__isset.enable_mapping_timestamp_tz && scan_params.enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = + scan_params.__isset.enable_mapping_varbinary && scan_params.enable_mapping_varbinary; if (delete_file.file_format == TFileFormatType::FORMAT_PARQUET) { + // Delete and data files must parse raw binary fields with the same scan-level mapping. *reader = std::make_unique( system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, - enable_mapping_timestamp_tz); + enable_mapping_timestamp_tz, enable_mapping_varbinary); } else { *reader = std::make_unique(system_properties, file_description, io_ctx, _scanner_profile, std::nullopt, diff --git a/be/src/format_v2/table_reader.cpp b/be/src/format_v2/table_reader.cpp index f7865aad63bc8d..2e0000c5fa2ace 100644 --- a/be/src/format_v2/table_reader.cpp +++ b/be/src/format_v2/table_reader.cpp @@ -787,10 +787,16 @@ Status TableReader::create_file_reader(std::unique_ptr* reader) { const bool enable_mapping_timestamp_tz = _scan_params != nullptr && _scan_params->__isset.enable_mapping_timestamp_tz && _scan_params->enable_mapping_timestamp_tz; + const bool enable_mapping_varbinary = _scan_params != nullptr && + _scan_params->__isset.enable_mapping_varbinary && + _scan_params->enable_mapping_varbinary; if (_format == FileFormat::PARQUET) { + // V2 must honor the scan contract directly; otherwise Hive STRING columns backed by an + // unannotated BYTE_ARRAY are silently exposed as VARBINARY and predicate bytes no longer + // match the table type. *reader = std::make_unique( _system_properties, _current_task->data_file, _io_ctx, _scanner_profile, - _global_rowid_context, enable_mapping_timestamp_tz); + _global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); return Status::OK(); } if (_format == FileFormat::ORC) { diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 493af2b29c81c6..239572d16b7fea 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -35,6 +35,7 @@ #include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" #include "format_v2/parquet/reader/native/column_reader.h" #include "format_v2/parquet/reader/native/decoder.h" +#include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" #include "io/fs/buffered_reader.h" @@ -1025,6 +1026,15 @@ TEST(ParquetV2NativeDecoderTest, CompactDeltaSkipKeepsScratchBounded) { EXPECT_LE(decoder->retained_scratch_bytes(), 4096 * sizeof(int32_t) + 1); } +TEST(ParquetV2NativeDecoderTest, DeltaPaddingBitCountIsWidenedBeforeCursorAdvance) { + int64_t padding_bits = 0; + ASSERT_TRUE(detail::checked_delta_padding_bits(64, std::numeric_limits::max(), + &padding_bits) + .ok()); + EXPECT_EQ(padding_bits, 64LL * std::numeric_limits::max()); + EXPECT_GT(padding_bits, std::numeric_limits::max()); +} + TEST(ParquetV2NativeDecoderTest, DeltaByteArraySkipsKeepScratchBounded) { constexpr size_t value_count = 1U << 16; std::vector<::parquet::ByteArray> values(value_count); @@ -1380,6 +1390,27 @@ TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption impossible_counts.data_page_header_v2.__set_repetition_levels_byte_length(0); impossible_counts.data_page_header_v2.__set_num_nulls(2); EXPECT_TRUE(parse_header(impossible_counts).is()); + + tparquet::PageHeader missing_layout; + missing_layout.type = tparquet::PageType::DATA_PAGE; + missing_layout.__set_compressed_page_size(0); + missing_layout.__set_uncompressed_page_size(0); + EXPECT_TRUE(parse_header(missing_layout).is()); + + auto swapped_layout = oversized_levels; + swapped_layout.type = tparquet::PageType::DATA_PAGE; + EXPECT_TRUE(parse_header(swapped_layout).is()); + + auto competing_layouts = negative; + competing_layouts.__set_compressed_page_size(0); + competing_layouts.__set_uncompressed_page_size(0); + competing_layouts.__isset.data_page_header_v2 = true; + competing_layouts.data_page_header_v2.__set_num_values(0); + competing_layouts.data_page_header_v2.__set_num_rows(0); + competing_layouts.data_page_header_v2.__set_num_nulls(0); + competing_layouts.data_page_header_v2.__set_repetition_levels_byte_length(0); + competing_layouts.data_page_header_v2.__set_definition_levels_byte_length(0); + EXPECT_TRUE(parse_header(competing_layouts).is()); } TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMismatch) { @@ -1673,6 +1704,7 @@ TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingData index_header.type = tparquet::PageType::INDEX_PAGE; index_header.__set_compressed_page_size(3); index_header.__set_uncompressed_page_size(3); + index_header.__set_index_page_header(tparquet::IndexPageHeader()); auto bytes = serialize_page(index_header, {1, 2, 3}); tparquet::PageHeader data_header; @@ -1703,7 +1735,8 @@ TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingData ParquetPageReadContext context(false, ""); ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, context); - ASSERT_TRUE(chunk_reader.init().ok()); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; EXPECT_EQ(chunk_reader.remaining_num_values(), 1); ASSERT_TRUE(chunk_reader.load_page_data().ok()); } diff --git a/be/test/format_v2/parquet/parquet_reader_control_test.cpp b/be/test/format_v2/parquet/parquet_reader_control_test.cpp index 2d187785bed723..5c434a346bd2c9 100644 --- a/be/test/format_v2/parquet/parquet_reader_control_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_control_test.cpp @@ -70,17 +70,26 @@ class CursorColumnReader final : public ParquetColumnReader { } void flush_profile() override { ++_profile_flushes; } + bool crossed_page_since_last_batch() override { + ++_page_crossing_checks; + return _crossed_page; + } + + void set_crossed_page(bool crossed_page) { _crossed_page = crossed_page; } int64_t cursor() const { return _cursor; } const std::vector& skip_lengths() const { return _skip_lengths; } const std::vector& read_lengths() const { return _read_lengths; } int profile_flushes() const { return _profile_flushes; } + int page_crossing_checks() const { return _page_crossing_checks; } private: int64_t _cursor = 0; std::vector _skip_lengths; std::vector _read_lengths; int _profile_flushes = 0; + bool _crossed_page = false; + int _page_crossing_checks = 0; }; GlobalRowLoacationV2 decode_rowid(const ColumnString& column, size_t row) { @@ -208,6 +217,24 @@ TEST(ParquetColumnReaderControlTest, SchedulerFlushesReaderProfilesAtBatchBounda EXPECT_EQ(reader_ptr->profile_flushes(), 1); } +TEST(ParquetColumnReaderControlTest, SchedulerOrsPageCrossingOncePerBatch) { + ParquetScanScheduler scheduler; + auto predicate_reader = std::make_unique(); + auto* predicate_ptr = predicate_reader.get(); + predicate_ptr->set_crossed_page(true); + scheduler._current_predicate_columns.emplace(0, std::move(predicate_reader)); + + auto lazy_reader = std::make_unique(); + auto* lazy_ptr = lazy_reader.get(); + lazy_ptr->set_crossed_page(true); + scheduler._current_non_predicate_columns.emplace(1, std::move(lazy_reader)); + + // Both readers are sampled even after the OR becomes true so their next batch starts cleanly. + EXPECT_TRUE(scheduler.finish_current_reader_batch_profiles()); + EXPECT_EQ(predicate_ptr->page_crossing_checks(), 1); + EXPECT_EQ(lazy_ptr->page_crossing_checks(), 1); +} + TEST(ParquetVirtualColumnReaderTest, RowPositionReadSkipAndInvalidArgs) { RowPositionColumnReader reader(100); EXPECT_EQ(reader.file_column_id(), format::ROW_POSITION_COLUMN_ID); diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index a90925f0eb58ec..c9239a152e5ab3 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -471,6 +471,14 @@ std::shared_ptr build_string_array(const std::vector& return finish_array(&builder); } +std::shared_ptr build_binary_array(const std::vector& values) { + arrow::BinaryBuilder builder; + for (const auto& value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_timestamp_array(const std::shared_ptr& type, const std::vector& values) { arrow::TimestampBuilder builder(type, arrow::default_memory_pool()); @@ -543,6 +551,18 @@ void write_parquet_file(const std::string& file_path, int64_t row_group_size = R row_group_size, builder.build())); } +void write_unannotated_binary_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("raw_bytes", arrow::binary(), false)}); + auto table = arrow::Table::Make(schema, {build_binary_array({"否", "是", "测试"})}); + auto file_result = arrow::io::FileOutputStream::Open(file_path); + ASSERT_TRUE(file_result.ok()) << file_result.status(); + std::shared_ptr out = *file_result; + ::parquet::WriterProperties::Builder builder; + builder.compression(::parquet::Compression::UNCOMPRESSED); + PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 3, + builder.build())); +} + void write_decimal_and_fixed_binary_parquet_file(const std::string& file_path) { auto decimal_type = arrow::decimal128(38, 6); auto fixed_type = arrow::fixed_size_binary(4); @@ -1234,7 +1254,7 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false) const { + bool is_immutable = false, bool enable_mapping_varbinary = false) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1245,7 +1265,7 @@ class NewParquetReaderTest : public testing::Test { file_description->is_immutable = is_immutable; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, - global_rowid_context, enable_mapping_timestamp_tz); + global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); } std::filesystem::path _test_dir; @@ -1270,6 +1290,39 @@ TEST_F(NewParquetReaderTest, GetSchemaReturnsFileLocalColumns) { EXPECT_EQ(remove_nullable(schema[1].type)->get_primitive_type(), TYPE_STRING); } +TEST_F(NewParquetReaderTest, RawByteArrayMappingFollowsV2ScanOption) { + write_unannotated_binary_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + auto string_reader = create_reader(); + const auto string_init_status = string_reader->init(&state); + ASSERT_TRUE(string_init_status.ok()) << string_init_status; + std::vector string_schema; + ASSERT_TRUE(string_reader->get_schema(&string_schema).ok()); + ASSERT_EQ(string_schema.size(), 1); + EXPECT_EQ(remove_nullable(string_schema[0].type)->get_primitive_type(), TYPE_STRING); + + RuntimeProfile binary_profile("raw_binary_mapping_profile"); + auto binary_reader = + create_reader(0, -1, &binary_profile, false, nullptr, std::nullopt, false, true); + const auto binary_init_status = binary_reader->init(&state); + ASSERT_TRUE(binary_init_status.ok()) << binary_init_status; + std::vector binary_schema; + ASSERT_TRUE(binary_reader->get_schema(&binary_schema).ok()); + ASSERT_EQ(binary_schema.size(), 1); + // The explicit VARBINARY mapping must remain available for scans whose table contract asks for it. + EXPECT_EQ(remove_nullable(binary_schema[0].type)->get_primitive_type(), TYPE_VARBINARY); + + auto request = std::make_shared(); + request->predicate_columns = {field_projection(0)}; + request->local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request->conjuncts.push_back(create_string_in_conjunct(0, {"是"})); + ASSERT_TRUE(binary_reader->open(request).ok()); + // A table-side STRING comparison may be rewritten through this VARBINARY file slot. Native + // dictionary pruning must leave the row group available for the mapping expression. + EXPECT_EQ(binary_profile.get_counter("RowGroupsFilteredByDictionary")->value(), 0); +} + // Scenario: Parquet is columnar and supports predicate/non-predicate split, nested projection and // file-layer pruning hints. The reader declares those scan-request capabilities by choosing // ParquetColumnMapper itself. diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 8ef1ad1143abf9..ba78775e85d80e 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -628,6 +628,107 @@ TEST(ParquetSchemaTest, NativeGroupEnumLogicalTypeIsRejected) { std::string::npos); } +TEST(ParquetSchemaTest, NativeStringAnnotationsAndTimeUnitsPreserveLogicalTypes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(8); + + auto binary_leaf = [](const std::string& name) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return leaf; + }; + auto logical_enum = binary_leaf("logical_enum"); + logical_enum.__set_logicalType(tparquet::LogicalType()); + logical_enum.logicalType.__set_ENUM(tparquet::EnumType()); + auto logical_bson = binary_leaf("logical_bson"); + logical_bson.__set_logicalType(tparquet::LogicalType()); + logical_bson.logicalType.__set_BSON(tparquet::BsonType()); + auto converted_enum = binary_leaf("converted_enum"); + converted_enum.__set_converted_type(tparquet::ConvertedType::ENUM); + auto converted_bson = binary_leaf("converted_bson"); + converted_bson.__set_converted_type(tparquet::ConvertedType::BSON); + + auto logical_time = [](const std::string& name, bool millis) { + tparquet::SchemaElement leaf; + leaf.__set_name(name); + leaf.__set_type(millis ? tparquet::Type::INT32 : tparquet::Type::INT64); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + leaf.__set_logicalType(tparquet::LogicalType()); + leaf.logicalType.__set_TIME(tparquet::TimeType()); + leaf.logicalType.TIME.__set_isAdjustedToUTC(false); + leaf.logicalType.TIME.__set_unit(tparquet::TimeUnit()); + if (millis) { + leaf.logicalType.TIME.unit.__set_MILLIS(tparquet::MilliSeconds()); + } else { + leaf.logicalType.TIME.unit.__set_MICROS(tparquet::MicroSeconds()); + } + return leaf; + }; + auto logical_millis = logical_time("logical_millis", true); + auto logical_micros = logical_time("logical_micros", false); + auto converted_millis = logical_time("converted_millis", true); + converted_millis.__isset.logicalType = false; + converted_millis.__set_converted_type(tparquet::ConvertedType::TIME_MILLIS); + auto converted_micros = logical_time("converted_micros", false); + converted_micros.__isset.logicalType = false; + converted_micros.__set_converted_type(tparquet::ConvertedType::TIME_MICROS); + + NativeFieldDescriptor descriptor; + ASSERT_TRUE(descriptor + .parse_from_thrift({root, logical_enum, logical_bson, converted_enum, + converted_bson, logical_millis, logical_micros, + converted_millis, converted_micros}) + .ok()); + for (size_t field = 0; field < 4; ++field) { + EXPECT_EQ(remove_nullable(descriptor.get_column(field)->data_type)->get_primitive_type(), + TYPE_STRING); + } + EXPECT_EQ(remove_nullable(descriptor.get_column(4)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(5)->data_type)->get_scale(), 6); + EXPECT_EQ(remove_nullable(descriptor.get_column(6)->data_type)->get_scale(), 3); + EXPECT_EQ(remove_nullable(descriptor.get_column(7)->data_type)->get_scale(), 6); +} + +TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { + auto valid_root = []() { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + return root; + }; + auto valid_leaf = []() { + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + return leaf; + }; + + NativeFieldDescriptor descriptor; + auto primitive_root = valid_root(); + primitive_root.__set_type(tparquet::Type::INT32); + EXPECT_FALSE(descriptor.parse_from_thrift({primitive_root, valid_leaf()}).ok()); + + auto repeated_root = valid_root(); + repeated_root.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + EXPECT_FALSE(descriptor.parse_from_thrift({repeated_root, valid_leaf()}).ok()); + + auto missing_repetition = valid_leaf(); + missing_repetition.__isset.repetition_type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_repetition}).ok()); + + auto dual_kind = valid_leaf(); + dual_kind.__set_num_children(1); + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), dual_kind}).ok()); + + auto missing_kind = valid_leaf(); + missing_kind.__isset.type = false; + EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_kind}).ok()); +} + TEST(ParquetSchemaTest, NativeSchemaRejectsUnboundedChildCountsBeforeAllocation) { tparquet::SchemaElement root; root.__set_name("schema"); diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index 77a43b02493d19..d6d99230e8f474 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -1096,5 +1096,35 @@ TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { *bloom_filter)); } +TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { + format::parquet::ParquetTypeDescriptor binary_type; + binary_type.physical_type = ::parquet::Type::BYTE_ARRAY; + + tparquet::Statistics max_only; + max_only.__set_max("III"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only)); + + tparquet::Statistics legacy_different; + legacy_different.__set_min("III"); + legacy_different.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_different)); + + tparquet::Statistics legacy_equal; + legacy_equal.__set_min("same"); + legacy_equal.__set_max("same"); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal)); + + tparquet::Statistics type_defined; + type_defined.__set_min_value("III"); + type_defined.__set_max_value("\xe6\x98\xaf"); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined)); + + tparquet::Statistics mixed_fields; + mixed_fields.__set_min_value("III"); + mixed_fields.__set_max("\xe6\x98\xaf"); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields)); +} + } // namespace } // namespace doris From c894604c1fc29b19797539e3a741edc6cae8d8f2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 10:10:44 +0800 Subject: [PATCH 18/34] fix: harden v2 parquet compatibility --- .../format_v2/parquet/native_schema_desc.cpp | 12 +- .../reader/native/column_chunk_reader.cpp | 19 +- .../parquet/reader/native/column_reader.cpp | 57 +++++- .../parquet/reader/native/page_reader.cpp | 4 + be/src/format_v2/table/hudi_reader.cpp | 26 ++- be/src/format_v2/table/hudi_reader.h | 11 ++ be/src/format_v2/table/paimon_reader.cpp | 25 ++- be/src/format_v2/table/paimon_reader.h | 11 ++ .../format_v2/parquet/native_decoder_test.cpp | 162 +++++++++++++++++- .../format_v2/parquet/parquet_schema_test.cpp | 4 + be/test/format_v2/table/hudi_reader_test.cpp | 61 +++++++ .../format_v2/table/paimon_reader_test.cpp | 62 +++++++ 12 files changed, 407 insertions(+), 47 deletions(-) diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index cc9ba4b403cb84..7ce1673a0ae6b4 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -107,14 +107,16 @@ static Status validate_native_schema_structure( if (!schema.__isset.repetition_type) { return Status::InvalidArgument("Schema element {} has no repetition type", pos); } - if (schema.__isset.type == schema.__isset.num_children) { - // Every non-root node is exactly one of primitive or group. Rejecting both missing and - // dual kind fields prevents the parser from guessing based on a default child count. + const bool has_children = schema.__isset.num_children && schema.num_children > 0; + if (schema.__isset.type == has_children) { + // Some legacy parquet-cpp files explicitly encode num_children=0 on primitive nodes. + // Only a positive count denotes a group, preserving strict rejection of real dual-kind + // nodes without dropping those otherwise valid files. return Status::InvalidArgument("Schema element {} has ambiguous primitive/group kind", pos); } - if (schema.__isset.num_children && schema.num_children <= 0) { - return Status::InvalidArgument("Group schema element {} has no children", pos); + if (schema.__isset.num_children && schema.num_children < 0) { + return Status::InvalidArgument("Schema element {} has a negative child count", pos); } const int children = num_children_node(schemas[pos]); if (children < 0 || static_cast(children) > schemas.size() - pos - 1) { diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 5dcbf46ccf9dde..63977164f25def 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -143,23 +143,12 @@ namespace { Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, const ParquetInt96Timestamp& value, const cctz::time_zone& timezone) { - static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; - static constexpr int64_t MICROS_PER_DAY = 86400000000LL; static constexpr int64_t MICROS_PER_SECOND = 1000000LL; - // Arrow normalized out-of-day INT96 nanos before the V2 native path replaced it. Preserve that - // file-compatibility invariant here; rejecting the raw nanos loses otherwise valid year-0 and - // pre-epoch timestamps used by existing Parquet files. - const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; - // The compatibility cast truncates the signed nanos field itself. Normalizing it to a positive - // time-of-day first changes negative out-of-day values by one microsecond. - const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; - if (timestamp_micros < std::numeric_limits::min() || - timestamp_micros > std::numeric_limits::max()) { - return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); - } - - const int64_t micros = static_cast(timestamp_micros); + int64_t micros = 0; + // Keep the fast ColumnDateTimeV2 path on the shared INT96 contract so plain, dictionary, and + // sparse selections cannot materialize an invalid nanos-of-day that SerDe would reject. + RETURN_IF_ERROR(parquet_int96_timestamp_micros(value, µs)); int64_t epoch_seconds = micros / MICROS_PER_SECOND; int64_t micros_of_second = micros % MICROS_PER_SECOND; if (micros_of_second < 0) { diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 11689dd5fb3104..d7fc92f2db9c98 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -1520,6 +1520,33 @@ Status MapColumnReader::read_column_data(ColumnPtr& doris_column, const DataType return Status::OK(); } + const auto parent_shape = [this](const ColumnReader& reader) { + std::vector> shape; + const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } + shape.reserve(rep_levels.size()); + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + // MAP siblings may have child-local collections and nullness. At this boundary, + // only the outer entry distribution and whether that entry exists must agree. + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); + } + } + return shape; + }; + if (UNLIKELY(_key_reader->get_rep_level().size() != _key_reader->get_def_level().size() || + _value_reader->get_rep_level().size() != _value_reader->get_def_level().size())) { + return Status::Corruption("Parquet map sibling level counts differ"); + } + if (UNLIKELY(parent_shape(*_key_reader) != parent_shape(*_value_reader))) { + return Status::Corruption("Parquet map key/value outer entry shapes differ"); + } + if (UNLIKELY(key_column->size() != value_column->size())) { return Status::Corruption("Parquet map key/value entry counts differ: {} vs {}", key_column->size(), value_column->size()); @@ -1602,17 +1629,23 @@ Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataT size_t not_missing_orig_column_size = 0; std::vector missing_column_idxs {}; std::vector skip_reading_column_idxs {}; - std::vector reference_parent_shape; + std::vector> reference_parent_shape; auto parent_shape = [this](const ColumnReader& reader) { - std::vector shape; + std::vector> shape; const auto& rep_levels = reader.get_rep_level(); + const auto& def_levels = reader.get_def_level(); + if (rep_levels.size() != def_levels.size()) { + return shape; + } shape.reserve(rep_levels.size()); - for (const level_t rep_level : rep_levels) { + for (size_t slot = 0; slot < rep_levels.size(); ++slot) { // Deeper repetitions belong to a nested child; only starts visible at this STRUCT's - // repeated-parent boundary determine how sibling values are paired. - if (rep_level <= _field_schema->repetition_level) { - shape.push_back(rep_level); + // boundary and the optional parent's presence determine how siblings are paired. + if (rep_levels[slot] <= _field_schema->repetition_level && + def_levels[slot] >= _field_schema->repeated_parent_def_level) { + shape.emplace_back(rep_levels[slot], + def_levels[slot] >= _field_schema->definition_level); } } return shape; @@ -1657,6 +1690,12 @@ Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataT &field_rows, &field_eof, is_dict_filter)); *read_rows = field_rows; *eof = field_eof; + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } reference_parent_shape = parent_shape(*_child_readers[file_name]); /* * Considering the issue in the `_read_nested_column` function where data may span across pages, leading @@ -1680,6 +1719,12 @@ Status StructColumnReader::read_column_data(ColumnPtr& doris_column, const DataT return Status::Corruption("Parquet struct child '{}' returned {} rows, expected {}", file_name, field_rows, *read_rows); } + if (UNLIKELY(_child_readers[file_name]->get_rep_level().size() != + _child_readers[file_name]->get_def_level().size())) { + return Status::Corruption( + "Parquet struct child '{}' has mismatched repetition/definition levels", + file_name); + } if (UNLIKELY(parent_shape(*_child_readers[file_name]) != reference_parent_shape)) { return Status::Corruption( "Parquet struct child '{}' has a different repeated-parent shape", diff --git a/be/src/format_v2/parquet/reader/native/page_reader.cpp b/be/src/format_v2/parquet/reader/native/page_reader.cpp index b679ed25ab9feb..f9216a244a4e20 100644 --- a/be/src/format_v2/parquet/reader/native/page_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/page_reader.cpp @@ -73,6 +73,10 @@ Status PageReader::_validate_page_header(uint32_t h matching_layout = has_index && !has_v1 && !has_v2 && !has_dictionary; break; default: + // Forward-compatible auxiliary pages have no known page-specific member. Their bounded + // payload can be skipped safely; accepting a known member here would let the enum and + // decoder layout disagree. + matching_layout = !has_v1 && !has_v2 && !has_dictionary && !has_index; break; } if (UNLIKELY(!matching_layout)) { diff --git a/be/src/format_v2/table/hudi_reader.cpp b/be/src/format_v2/table/hudi_reader.cpp index f66be726fdcfee..d05cfe4b03e357 100644 --- a/be/src/format_v2/table/hudi_reader.cpp +++ b/be/src/format_v2/table/hudi_reader.cpp @@ -73,13 +73,9 @@ Status HudiHybridReader::init(format::TableReadOptions&& options) { } Status HudiHybridReader::prepare_split(const format::SplitReadOptions& options) { - { - // Child readers use these same counters, so time only dispatch/creation here and end the - // outer scopes before invoking the child to preserve single-counted totals. - SCOPED_TIMER(_profile.total_timer); - SCOPED_TIMER(_profile.prepare_split_timer); - RETURN_IF_ERROR(_ensure_current_split_reader(options)); - } + // A newly selected child initializes against the same scanner profile. Keep hybrid dispatch + // outside those shared counters so first-split initialization is counted exactly once. + RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); } @@ -133,7 +129,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt DORIS_CHECK(_scan_params != nullptr); if (_is_jni_split(*_scan_params, options.current_range)) { if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -141,7 +145,15 @@ Status HudiHybridReader::_ensure_current_split_reader(const format::SplitReadOpt format::FileFormat file_format; RETURN_IF_ERROR(_to_file_format(*_scan_params, options.current_range, &file_format)); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::hudi::HudiReader::create_unique(); + } +#else _native_reader = format::hudi::HudiReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); diff --git a/be/src/format_v2/table/hudi_reader.h b/be/src/format_v2/table/hudi_reader.h index 77a71cd79ba166..d81e8c80e93ed3 100644 --- a/be/src/format_v2/table/hudi_reader.h +++ b/be/src/format_v2/table/hudi_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include #include @@ -73,6 +74,12 @@ class HudiHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -88,6 +95,10 @@ class HudiHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle native parquet/orc splits std::unique_ptr _jni_reader; // handle MOR JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::hudi diff --git a/be/src/format_v2/table/paimon_reader.cpp b/be/src/format_v2/table/paimon_reader.cpp index 869b2b1294a675..942fbb234269fc 100644 --- a/be/src/format_v2/table/paimon_reader.cpp +++ b/be/src/format_v2/table/paimon_reader.cpp @@ -100,12 +100,9 @@ Status PaimonHybridReader::init(format::TableReadOptions&& options) { } Status PaimonHybridReader::prepare_split(const format::SplitReadOptions& options) { - { - // End the outer dispatch scopes before the selected child enters the shared counters. - SCOPED_TIMER(_profile.total_timer); - SCOPED_TIMER(_profile.prepare_split_timer); - RETURN_IF_ERROR(_ensure_current_split_reader(options)); - } + // Child initialization uses the scanner profile too; hybrid dispatch must not nest the same + // timer around the first native or JNI child and double-count that initialization. + RETURN_IF_ERROR(_ensure_current_split_reader(options)); DORIS_CHECK(_current_split_reader != nullptr); return _current_split_reader->prepare_split(options); } @@ -159,7 +156,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO if (_is_jni_split(options.current_range)) { DCHECK(options.current_split_format == format::FileFormat::JNI); if (_jni_reader == nullptr) { +#ifdef BE_TEST + if (_test_jni_reader_factory) { + _jni_reader = _test_jni_reader_factory(); + } else { + _jni_reader = std::make_unique(); + } +#else _jni_reader = std::make_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_jni_reader.get(), format::FileFormat::JNI)); } _current_split_reader = _jni_reader.get(); @@ -170,7 +175,15 @@ Status PaimonHybridReader::_ensure_current_split_reader(const format::SplitReadO DCHECK(file_format == format::FileFormat::PARQUET || file_format == format::FileFormat::ORC); if (_native_reader == nullptr) { +#ifdef BE_TEST + if (_test_native_reader_factory) { + _native_reader = _test_native_reader_factory(); + } else { + _native_reader = format::paimon::PaimonReader::create_unique(); + } +#else _native_reader = format::paimon::PaimonReader::create_unique(); +#endif RETURN_IF_ERROR(_init_child_reader(_native_reader.get(), file_format)); } _current_split_reader = _native_reader.get(); diff --git a/be/src/format_v2/table/paimon_reader.h b/be/src/format_v2/table/paimon_reader.h index 634f2bd7a6400c..ad29075fed1f08 100644 --- a/be/src/format_v2/table/paimon_reader.h +++ b/be/src/format_v2/table/paimon_reader.h @@ -17,6 +17,7 @@ #pragma once +#include #include #include "format_v2/table_reader.h" @@ -84,6 +85,12 @@ class PaimonHybridReader final : public format::TableReader { std::pair TEST_child_batch_sizes() const { return {_native_reader->TEST_batch_size(), _jni_reader->TEST_batch_size()}; } + void TEST_set_child_reader_factories( + std::function()> native_factory, + std::function()> jni_factory) { + _test_native_reader_factory = std::move(native_factory); + _test_jni_reader_factory = std::move(jni_factory); + } #endif private: @@ -96,6 +103,10 @@ class PaimonHybridReader final : public format::TableReader { std::unique_ptr _native_reader; // handle parquet/orc native splits std::unique_ptr _jni_reader; // handle serialized JNI splits format::TableReader* _current_split_reader = nullptr; +#ifdef BE_TEST + std::function()> _test_native_reader_factory; + std::function()> _test_jni_reader_factory; +#endif }; } // namespace doris::format::paimon diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 239572d16b7fea..fc0c05ed611841 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -32,6 +32,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_struct.h" +#include "core/data_type_serde/parquet_timestamp.h" #include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" #include "format_v2/parquet/reader/native/column_reader.h" #include "format_v2/parquet/reader/native/decoder.h" @@ -300,6 +301,54 @@ Status load_malformed_nested_page(tparquet::PageHeader header, const std::vector return chunk_reader.load_page_nested_rows(rep_levels, 1, &result_rows, &cross_page); } +Status materialize_plain_int96(const std::vector& values, + const std::vector& filter_values = {}) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__set_uncompressed_page_size(values.size() * sizeof(ParquetInt96Timestamp)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(values.size()); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto bytes = serialize_page( + header, std::vector(reinterpret_cast(values.data()), + reinterpret_cast(values.data()) + + values.size() * sizeof(ParquetInt96Timestamp))); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT96); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(values.size()); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT96; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, values.size(), + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT96; + static const auto utc = cctz::utc_time_zone(); + decode_context.timezone = &utc; + ParquetMaterializationState state; + state.enable_strict_mode = true; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.empty() ? nullptr : filter_values.data(), + filter_values.size(), false)); + ColumnSelectVector select_vector; + const std::vector run_length_null_map {static_cast(values.size()), 0}; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, values.size(), nullptr, &filter, 0)); + return chunk_reader.materialize_values(column, *type.get_serde(), decode_context, state, + select_vector); +} + TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { int32_t dictionary_length = 0; auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); @@ -496,6 +545,14 @@ TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) integer_field, make_nullable(std::make_shared()), false)); } +TEST(ParquetV2NativeDecoderTest, FastInt96RejectsInvalidNanosOfDay) { + EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).is()); + EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}) + .is()); + EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}) + .is()); +} + TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { DataTypePtr datetime_type = make_nullable(std::make_shared(6)); NativeFieldSchema local_field; @@ -1217,7 +1274,8 @@ TEST(ParquetV2NativeDecoderTest, ByteStreamSplitRejectsPartialRowsAtPageBoundary Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t value_rows, size_t key_values, size_t value_values, std::vector key_rep_levels, std::vector value_rep_levels, - bool value_eof) { + bool value_eof, std::vector key_def_levels = {}, + std::vector value_def_levels = {}) { auto value_type = make_nullable(std::make_shared()); auto map_type = std::make_shared(key_type, value_type); ColumnPtr column = map_type->create_column(); @@ -1228,12 +1286,17 @@ Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t va field.repetition_level = 1; field.repeated_parent_def_level = 0; - auto key_reader = std::make_unique(key_rows, key_values, true, - std::move(key_rep_levels), - std::vector(key_values, 1)); - auto value_reader = std::make_unique( - value_rows, value_values, value_eof, std::move(value_rep_levels), - std::vector(value_values, 1)); + if (key_def_levels.empty()) { + key_def_levels.assign(key_rep_levels.size(), 1); + } + if (value_def_levels.empty()) { + value_def_levels.assign(value_rep_levels.size(), 1); + } + auto key_reader = std::make_unique( + key_rows, key_values, true, std::move(key_rep_levels), std::move(key_def_levels)); + auto value_reader = std::make_unique(value_rows, value_values, value_eof, + std::move(value_rep_levels), + std::move(value_def_levels)); MapColumnReader reader(scripted_row_ranges(), key_rows, nullptr, nullptr); RETURN_IF_ERROR(reader.init(std::move(key_reader), std::move(value_reader), &field)); @@ -1248,7 +1311,19 @@ Status read_scripted_map(const DataTypePtr& key_type, size_t key_rows, size_t va TEST(ParquetV2NativeDecoderTest, MapReaderUsesKeyShapeForNestedValues) { auto int_type = std::make_shared(); - EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 0}, true).ok()); + EXPECT_TRUE(read_scripted_map(int_type, 1, 1, 2, 2, {0, 1}, {0, 2, 1}, true).ok()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryDistribution) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 4, 4, {0, 1, 0, 1}, {0, 1, 1, 0}, true) + .is()); +} + +TEST(ParquetV2NativeDecoderTest, MapReaderRejectsShiftedEntryPresence) { + auto int_type = std::make_shared(); + EXPECT_TRUE(read_scripted_map(int_type, 2, 2, 2, 2, {0, 0}, {0, 0}, true, {0, 1}, {1, 0}) + .is()); } TEST(ParquetV2NativeDecoderTest, ComplexReadersRejectMalformedSiblingCounts) { @@ -1322,6 +1397,37 @@ TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedRepeatedParentShape) .is()); } +TEST(ParquetV2NativeDecoderTest, StructReaderRejectsShiftedOptionalParentPresence) { + auto int_type = std::make_shared(); + auto struct_type = + std::make_shared(DataTypes {int_type, int_type}, Strings {"a", "b"}); + ColumnPtr column = struct_type->create_column(); + NativeFieldSchema field; + field.name = "s"; + field.data_type = struct_type; + field.definition_level = 1; + field.children.resize(2); + field.children[0].name = "a"; + field.children[1].name = "b"; + + std::unordered_map> children; + children["a"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {0, 1}); + children["b"] = std::make_unique(2, 2, true, std::vector {0, 0}, + std::vector {1, 0}); + StructColumnReader reader(scripted_row_ranges(), 2, nullptr, nullptr); + ASSERT_TRUE(reader.init(std::move(children), &field).ok()); + auto root = std::make_shared(); + root->add_child("a", "a", std::make_shared()); + root->add_child("b", "b", std::make_shared()); + FilterMap filter; + size_t read_rows = 0; + bool eof = false; + EXPECT_TRUE( + reader.read_column_data(column, struct_type, root, filter, 2, &read_rows, &eof, false) + .is()); +} + TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { constexpr size_t value_count = 1UL << 20; std::vector encoded(value_count * sizeof(float), 0); @@ -1741,6 +1847,46 @@ TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsIndexPageBeforeInitializingData ASSERT_TRUE(chunk_reader.load_page_data().ok()); } +TEST(ParquetV2NativeDecoderTest, ColumnChunkSkipsUnknownAuxiliaryPage) { + tparquet::PageHeader unknown_header; + unknown_header.type = static_cast(127); + unknown_header.__set_compressed_page_size(3); + unknown_header.__set_uncompressed_page_size(3); + auto bytes = serialize_page(unknown_header, {1, 2, 3}); + + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(sizeof(int32_t)); + data_header.__set_uncompressed_page_size(sizeof(int32_t)); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(1); + data_header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + const int32_t value = 42; + auto data_bytes = serialize_page( + data_header, + std::vector(reinterpret_cast(&value), + reinterpret_cast(&value) + sizeof(value))); + bytes.insert(bytes.end(), data_bytes.begin(), data_bytes.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(1); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, + context); + const auto init_status = chunk_reader.init(); + ASSERT_TRUE(init_status.ok()) << init_status; + EXPECT_EQ(chunk_reader.remaining_num_values(), 1); +} + TEST(ParquetV2NativeDecoderTest, LegacyDataPageV2OverridesFalseCompressedFlag) { tparquet::PageHeader header; header.type = tparquet::PageType::DATA_PAGE_V2; diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index ba78775e85d80e..98cf963b30080a 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -724,6 +724,10 @@ TEST(ParquetSchemaTest, NativeSchemaRejectsAmbiguousKindsAndMissingRepetition) { dual_kind.__set_num_children(1); EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), dual_kind}).ok()); + auto legacy_zero_children = valid_leaf(); + legacy_zero_children.__set_num_children(0); + EXPECT_TRUE(descriptor.parse_from_thrift({valid_root(), legacy_zero_children}).ok()); + auto missing_kind = valid_leaf(); missing_kind.__isset.type = false; EXPECT_FALSE(descriptor.parse_from_thrift({valid_root(), missing_kind}).ok()); diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index 2dd001469d8a66..bdda544c93c792 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -24,9 +24,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -122,6 +124,23 @@ TTableFormatFileDesc hudi_table_format_desc(std::optional schema_id) { return table_format_params; } +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + // Scenario: FileScannerV2 Hudi native reader uses the split schema id to annotate the physical // file schema before TableColumnMapper runs. This keeps schema-evolved Hudi files on field-id // mapping, including renamed nested children. @@ -300,5 +319,47 @@ TEST(HudiHybridReaderTest, NativeCountStarReportsMetadataRowsThroughHybridReader std::filesystem::remove_all(test_dir); } +TEST(HudiHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + TFileScanRangeParams scan_params; + scan_params.__set_format_type(TFileFormatType::FORMAT_PARQUET); + hudi::HudiHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = nullptr, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileFormatType::type thrift_format) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range.__set_format_type(thrift_format); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, TFileFormatType::FORMAT_PARQUET); + verify_first_split(FileFormat::JNI, TFileFormatType::FORMAT_JNI); +} + } // namespace } // namespace doris::format diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index fbb3c79d5619b6..3c85eb4632f86c 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -23,11 +23,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include "core/assert_cast.h" @@ -60,6 +62,23 @@ namespace doris::format { namespace { +class SlowInitTableReader final : public TableReader { +public: + Status init(TableReadOptions&& options) override { + RETURN_IF_ERROR(TableReader::init(std::move(options))); + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.init_timer); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + return Status::OK(); + } + + Status prepare_split(const SplitReadOptions&) override { + SCOPED_TIMER(_profile.total_timer); + SCOPED_TIMER(_profile.prepare_split_timer); + return Status::OK(); + } +}; + DataTypePtr table_type(const DataTypePtr& type) { return type->is_nullable() ? type : make_nullable(type); } @@ -763,6 +782,49 @@ TEST(PaimonHybridReaderTest, DispatchesNativeThenJniSplitToMatchingReader) { ASSERT_TRUE(reader.close().ok()); } +TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { + RuntimeProfile profile("test_profile"); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + auto scan_params = make_local_parquet_scan_params(); + paimon::PaimonHybridReader reader; + ASSERT_TRUE(reader.init({ + .projected_columns = {}, + .conjuncts = {}, + .format = FileFormat::PARQUET, + .scan_params = &scan_params, + .io_ctx = nullptr, + .runtime_state = &state, + .scanner_profile = &profile, + .file_slot_descs = nullptr, + .push_down_agg_type = TPushAggOp::NONE, + .condition_cache_digest = 0, + }) + .ok()); + reader.TEST_set_child_reader_factories([] { return std::make_unique(); }, + [] { return std::make_unique(); }); + + auto* total = profile.get_counter("TableReader"); + auto* init = profile.get_counter("InitTime"); + ASSERT_NE(total, nullptr); + ASSERT_NE(init, nullptr); + auto verify_first_split = [&](FileFormat format, TFileRangeDesc range) { + SplitReadOptions split; + split.current_split_format = format; + split.current_range = std::move(range); + const int64_t total_before = total->value(); + const int64_t init_before = init->value(); + ASSERT_TRUE(reader.prepare_split(split).ok()); + const int64_t total_delta = total->value() - total_before; + const int64_t init_delta = init->value() - init_before; + EXPECT_GE(init_delta, std::chrono::milliseconds(25).count() * 1000 * 1000); + // A nested hybrid timer would add the 30 ms child init to total a second time. + EXPECT_LT(total_delta - init_delta, std::chrono::milliseconds(15).count() * 1000 * 1000); + }; + verify_first_split(FileFormat::PARQUET, + make_paimon_native_range(TFileFormatType::FORMAT_PARQUET)); + verify_first_split(FileFormat::JNI, make_paimon_jni_range()); +} + TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ From 898801d57435839000f6c30f0fc1d6ff200d9d6d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 13:02:36 +0800 Subject: [PATCH 19/34] fix: harden native parquet v2 compatibility --- .../parquet/parquet_file_context.cpp | 86 +++++++----- .../format_v2/parquet/parquet_file_context.h | 8 ++ .../format_v2/parquet/parquet_statistics.cpp | 61 ++++++++- be/src/format_v2/parquet/parquet_statistics.h | 5 + .../reader/native/column_chunk_reader.cpp | 48 +++++-- .../reader/native/column_chunk_reader.h | 3 + .../format_v2/parquet/native_decoder_test.cpp | 61 +++++++-- .../format_v2/parquet/parquet_reader_test.cpp | 73 +++++++++- .../parquet/parquet_statistics_test.cpp | 127 ++++++++++++++++++ 9 files changed, 415 insertions(+), 57 deletions(-) diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 1f804d6ba6b8a2..1dc545afa3934d 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -52,6 +52,8 @@ namespace doris::format::parquet { +constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; + NativeParquetMetadata::NativeParquetMetadata(tparquet::FileMetaData metadata, size_t parsed_size) : _metadata(std::move(metadata)), _parsed_size(parsed_size) { ExecEnv::GetInstance()->parquet_meta_tracker()->consume(get_mem_size()); @@ -170,6 +172,34 @@ size_t ParquetPageCacheRangeDirectory::size() const { return _indexes.size(); } +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit) { + if (file_size < V2_PARQUET_FOOTER_SIZE || + serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { + return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, + file_size); + } + if (serialized_size > metadata_size_limit) { + return Status::Corruption("Parquet v2 footer size {} exceeds metadata limit {}", + serialized_size, metadata_size_limit); + } + return Status::OK(); +} + +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable) { + const int64_t mtime = description_mtime != 0 ? description_mtime : reader_mtime; + if (mtime == 0 && !is_immutable) { + // Unknown version is not a stable identity: an overwrite can preserve path and size while + // changing both footer semantics and page bytes. + return {}; + } + const int64_t file_size = description_file_size >= 0 ? description_file_size : reader_file_size; + return fmt::format("{}::{}::mtime={}::size={}", fs_name, path, mtime, file_size); +} + bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length) { if (offset < 0 || length <= 0 || length > MAX_SERIALIZED_PARQUET_INDEX_BYTES || static_cast(offset) > file_size) { @@ -288,7 +318,6 @@ detail::ParquetPageCacheRangeDirectory& cached_page_range_directory() { } constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; -constexpr size_t V2_PARQUET_FOOTER_SIZE = 8; constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; Status parse_native_parquet_footer(io::FileReaderSPtr file, @@ -320,12 +349,13 @@ Status parse_native_parquet_footer(io::FileReaderSPtr file, const uint32_t serialized_size = decode_fixed32_le(tail.data() + tail.size() - V2_PARQUET_FOOTER_SIZE); - if (serialized_size > file_size - V2_PARQUET_FOOTER_SIZE) { - // Footer lengths are untrusted. Validate before subtraction/allocation so a malformed - // small file cannot redirect the v2 reader or request an oversized metadata buffer. - return Status::Corruption("Parquet v2 footer size {} exceeds file size {}", serialized_size, - file_size); - } + // The configured Thrift message ceiling also bounds this file-controlled allocation. Keep the + // check before both allocation and the optional second read so a sparse file cannot force a + // process-sized metadata buffer merely by advertising a large footer. + const size_t metadata_size_limit = + static_cast(std::max(config::thrift_max_message_size, 0)); + RETURN_IF_ERROR( + detail::validate_native_footer_size(serialized_size, file_size, metadata_size_limit)); std::vector serialized_metadata(serialized_size); if (serialized_size <= tail.size() - V2_PARQUET_FOOTER_SIZE) { const auto* metadata_start = @@ -356,21 +386,10 @@ Status parse_native_parquet_footer(io::FileReaderSPtr file, std::string build_page_cache_file_key(const io::FileReader& file_reader, const io::FileDescription& file_description) { - const int64_t mtime = - file_description.mtime != 0 ? file_description.mtime : file_reader.mtime(); - if (mtime == 0 && !file_description.is_immutable) { - // mtime == 0 means "unknown version", not the Unix epoch. V1 historically caches such a - // file under path::0, but copying that behavior for every V2 file is unsafe: a mutable file - // can be overwritten with different bytes while retaining both its path and size, causing - // process-global page cache entries to return stale data. Only callers that explicitly - // guarantee path immutability may use the mtime=0 cache key below. - return {}; - } - const int64_t file_size = file_description.file_size >= 0 - ? file_description.file_size - : static_cast(file_reader.size()); - return fmt::format("{}::{}::mtime={}::size={}", file_description.fs_name, - file_reader.path().native(), mtime, file_size); + return detail::build_native_file_cache_key( + file_description.fs_name, file_reader.path().native(), file_description.mtime, + file_reader.mtime(), file_description.file_size, + static_cast(file_reader.size()), file_description.is_immutable); } class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { @@ -727,16 +746,21 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont } native_io_ctx = io_ctx; - // V2 owns its footer payload and cache identity. Mapping flags affect the parsed schema, and a - // distinct suffix prevents a v1 FileMetaData value from being cast as the v2-owned type. + // Footer and page bytes must use the same stable identity. In particular, fs_name separates + // identical HDFS paths from different nameservices, while an unknown mutable version bypasses + // both caches rather than reusing an overwritten file of the same size. auto* meta_cache = ExecEnv::GetInstance()->file_meta_cache(); - auto meta_cache_key = FileMetaCache::get_key(native_file, file_description); - meta_cache_key.append("\0v2", 3); - // Schema mapping is part of the cached value, so both flags must participate in its identity. - meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); - meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); + auto meta_cache_key = build_page_cache_file_key(*native_file, file_description); + const bool has_stable_meta_cache_identity = !meta_cache_key.empty(); + if (has_stable_meta_cache_identity) { + // The discriminator prevents a v1 metadata value from being cast as the v2-owned type; + // schema mapping flags participate because they change the cached native schema tree. + meta_cache_key.append("\0v2", 3); + meta_cache_key.push_back(static_cast(enable_mapping_varbinary)); + meta_cache_key.push_back(static_cast(enable_mapping_timestamp_tz)); + } size_t native_footer_size = 0; - if (meta_cache != nullptr && meta_cache->enabled() && + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled() && meta_cache->lookup(meta_cache_key, &native_meta_cache_handle)) { native_metadata = native_meta_cache_handle.data(); ++native_footer_cache_hits; @@ -745,7 +769,7 @@ Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOCont native_file, &native_metadata_owner, &native_footer_size, io_ctx, enable_mapping_varbinary, enable_mapping_timestamp_tz)); ++native_footer_read_calls; - if (meta_cache != nullptr && meta_cache->enabled()) { + if (has_stable_meta_cache_identity && meta_cache != nullptr && meta_cache->enabled()) { meta_cache->insert(meta_cache_key, native_metadata_owner.release(), &native_meta_cache_handle); native_metadata = native_meta_cache_handle.data(); diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index ebac63b3e75993..add2b8e18f6fa2 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -129,6 +129,14 @@ class ParquetPageCacheRangeDirectory { inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; +Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, + size_t metadata_size_limit); + +std::string build_native_file_cache_key(std::string_view fs_name, std::string_view path, + int64_t description_mtime, int64_t reader_mtime, + int64_t description_file_size, int64_t reader_file_size, + bool is_immutable); + bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t length); bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index 214e7988bd1768..c0bb4ef10b4b67 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -535,6 +535,44 @@ class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { const ::parquet::BloomFilter& _bloom_filter; }; +class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { +public: + NativeParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, + const segment_v2::BloomFilter& bloom_filter) + : _column_schema(column_schema), _bloom_filter(bloom_filter) {} + + void add_bytes(const char*, size_t) override { DORIS_CHECK(false); } + + bool test_bytes(const char* buf, size_t size) const override { + if (buf == nullptr || + _column_schema.type_descriptor.physical_type != ::parquet::Type::INT32) { + return _bloom_filter.test_bytes(buf, size); + } + const auto logical_value = load_predicate_integral_value(buf, size); + if (!logical_value.has_value()) { + return true; + } + const auto physical_value = convert_logical_integer_to_physical_int32( + _column_schema.type_descriptor, *logical_value); + if (!physical_value.has_value()) { + return false; + } + // Native file Bloom bytes are hashed from the Parquet physical carrier, not the wider + // Doris logical literal used by VExpr (for example UINT32 is exposed as BIGINT). + return _bloom_filter.test_bytes(reinterpret_cast(&*physical_value), + sizeof(*physical_value)); + } + + void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } + bool has_null() const override { return false; } + void add_hash(uint64_t) override { DORIS_CHECK(false); } + bool test_hash(uint64_t hash) const override { return _bloom_filter.test_hash(hash); } + +private: + const ParquetColumnSchema& _column_schema; + const segment_v2::BloomFilter& _bloom_filter; +}; + bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { if (!bloom_logical_type_supported(column_schema)) { return false; @@ -981,6 +1019,21 @@ bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& colu return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); } +bool ParquetStatisticsUtils::NativeBloomFilterExcludes( + const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter) { + if (!bloom_filter_supported(column_schema)) { + return false; + } + NativeParquetBloomFilterAdapter adapter(column_schema, bloom_filter); + BloomFilterEvalContext ctx; + ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { + .data_type = column_schema.type, + .bloom_filter = &adapter, + }); + return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; +} + namespace { ParquetRowGroupPruneReason dictionary_prune_reason( @@ -1458,12 +1511,8 @@ ParquetRowGroupPruneReason native_bloom_filter_prune_reason( if (!status.ok() || bloom_filter == nullptr) { continue; } - BloomFilterEvalContext ctx; - ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { - .data_type = column_schema->type, - .bloom_filter = bloom_filter.get(), - }); - if (VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch) { + if (ParquetStatisticsUtils::NativeBloomFilterExcludes(*column_schema, slot_index, conjuncts, + *bloom_filter)) { return ParquetRowGroupPruneReason::BLOOM_FILTER; } } diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index aa1f3b614f0d72..ca47b343bcc3f9 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -47,6 +47,7 @@ class time_zone; namespace doris { class RuntimeState; namespace segment_v2 { +class BloomFilter; struct ZoneMap; } // namespace segment_v2 } // namespace doris @@ -167,6 +168,10 @@ struct ParquetStatisticsUtils { static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, const VExprContextSPtrs& conjuncts, const ::parquet::BloomFilter& bloom_filter); + + static bool NativeBloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, + const VExprContextSPtrs& conjuncts, + const segment_v2::BloomFilter& bloom_filter); }; Status select_row_groups_by_metadata( diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 63977164f25def..99b80d81cb8aa1 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -63,6 +63,24 @@ bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cac return session_cache_enabled && !storage_cache_disabled && cache_available && header_available; } +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed) { + const bool is_v2 = header.__isset.data_page_header_v2; + const bool page_is_compressed = + codec != tparquet::CompressionCodec::UNCOMPRESSED && + (!is_v2 || header.data_page_header_v2.is_compressed || data_page_v2_always_compressed); + if (!page_is_compressed && + UNLIKELY(header.compressed_page_size != header.uncompressed_page_size)) { + // An uncompressed payload has one physical representation, so accepting two lengths makes + // cold reads and decompressed cache hits consume different byte boundaries. + return Status::Corruption( + "Uncompressed Parquet page sizes differ: compressed={}, uncompressed={}", + header.compressed_page_size, header.uncompressed_page_size); + } + return Status::OK(); +} + ParquetReaderCompat parquet_reader_compat(const std::string& created_by) { if (created_by.empty()) { return {}; @@ -143,12 +161,23 @@ namespace { Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, const ParquetInt96Timestamp& value, const cctz::time_zone& timezone) { + static constexpr int32_t JULIAN_EPOCH_OFFSET_DAYS = 2440588; + static constexpr int64_t MICROS_PER_DAY = 86400000000LL; static constexpr int64_t MICROS_PER_SECOND = 1000000LL; - int64_t micros = 0; - // Keep the fast ColumnDateTimeV2 path on the shared INT96 contract so plain, dictionary, and - // sparse selections cannot materialize an invalid nanos-of-day that SerDe would reject. - RETURN_IF_ERROR(parquet_int96_timestamp_micros(value, µs)); + // Arrow normalized out-of-day INT96 nanos before the native V2 path replaced it. Preserve that + // legacy-writer compatibility here; rejecting the carrier loses valid pre-epoch/year-0 values + // already accepted by Doris external-table scans. + const __int128 days = static_cast<__int128>(value.julian_day) - JULIAN_EPOCH_OFFSET_DAYS; + // Truncate the signed nanos field before day normalization. Flooring a negative value first + // changes the historical result by one microsecond. + const __int128 timestamp_micros = days * MICROS_PER_DAY + value.nanos_of_day / 1000; + if (timestamp_micros < std::numeric_limits::min() || + timestamp_micros > std::numeric_limits::max()) { + return Status::DataQualityError("Parquet INT96 timestamp overflows microseconds"); + } + + const int64_t micros = static_cast(timestamp_micros); int64_t epoch_seconds = micros / MICROS_PER_SECOND; int64_t micros_of_second = micros % MICROS_PER_SECOND; if (micros_of_second < 0) { @@ -601,6 +630,8 @@ Status ColumnChunkReader::load_page_data() { const tparquet::PageHeader* header = nullptr; RETURN_IF_ERROR(_page_reader->get_page_header(&header)); + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); int32_t uncompressed_size = header->uncompressed_page_size; bool page_loaded = false; @@ -833,13 +864,8 @@ Status ColumnChunkReader::_decode_dict_page() { // Prepare dictionary data int32_t uncompressed_size = header->uncompressed_page_size; - if (_block_compress_codec == nullptr && - UNLIKELY(header->compressed_page_size != uncompressed_size)) { - // UNCOMPRESSED pages use the compressed size as their physical copy length. - return Status::Corruption( - "Uncompressed Parquet dictionary sizes differ: compressed={}, uncompressed={}", - header->compressed_page_size, uncompressed_size); - } + RETURN_IF_ERROR(validate_uncompressed_page_sizes( + *header, _metadata.codec, _page_read_ctx.data_page_v2_always_compressed)); auto dict_data = make_unique_buffer(uncompressed_size); bool dict_loaded = false; diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index e35e6b675c730c..f773b6c1b1bc0f 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -67,6 +67,9 @@ bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunk int64_t data_page_offset, int64_t row_count); bool can_prepare_page_cache_payload(bool session_cache_enabled, bool storage_cache_disabled, bool cache_available, bool header_available); +Status validate_uncompressed_page_sizes(const tparquet::PageHeader& header, + tparquet::CompressionCodec::type codec, + bool data_page_v2_always_compressed); struct ColumnChunkReaderStatistics { int64_t decompress_time = 0; diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index fc0c05ed611841..7634d714e8d457 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -245,12 +245,25 @@ std::vector serialize_page(tparquet::PageHeader header, } Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, - tparquet::CompressionCodec::type codec) { + tparquet::CompressionCodec::type codec, bool preload_page_cache = false) { std::vector bytes; ThriftSerializer serializer(/*compact=*/true, 128); RETURN_IF_ERROR(serializer.serialize(&header, &bytes)); bytes.insert(bytes.end(), payload.begin(), payload.end()); const size_t chunk_size = bytes.size(); + const std::string page_cache_file_key = "native-scripted-page-" + + std::to_string(static_cast(header.type)) + "-" + + std::to_string(header.compressed_page_size) + "-" + + std::to_string(header.uncompressed_page_size); + if (preload_page_cache) { + auto* page = new DataPage(bytes.size(), true, segment_v2::DATA_PAGE); + memcpy(page->data(), bytes.data(), bytes.size()); + page->reset_size(bytes.size()); + PageCacheHandle handle; + StoragePageCache::instance()->insert( + StoragePageCache::CacheKey(page_cache_file_key, chunk_size, 0), page, &handle, + segment_v2::DATA_PAGE); + } MemoryBufferedReader reader(std::move(bytes)); tparquet::ColumnChunk chunk; @@ -268,7 +281,7 @@ Status load_scripted_page(tparquet::PageHeader header, const std::vector chunk_reader(&reader, &chunk, &field, nullptr, 1, nullptr, context); RETURN_IF_ERROR(chunk_reader.init()); @@ -545,12 +558,10 @@ TEST(ParquetV2NativeDecoderTest, NonStrictLegacyTimestampsKeepDefaultOnOverflow) integer_field, make_nullable(std::make_shared()), false)); } -TEST(ParquetV2NativeDecoderTest, FastInt96RejectsInvalidNanosOfDay) { - EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).is()); - EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}) - .is()); - EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}) - .is()); +TEST(ParquetV2NativeDecoderTest, FastInt96NormalizesLegacyOutOfDayNanos) { + EXPECT_TRUE(materialize_plain_int96({{-1, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{86400000000000LL, 2440588}}).ok()); + EXPECT_TRUE(materialize_plain_int96({{0, 2440588}, {-1, 2440588}}, {0, 1}).ok()); } TEST(ParquetV2NativeDecoderTest, NonStrictLocalTimestampDefaultsBecomeNull) { @@ -1719,6 +1730,40 @@ TEST(ParquetV2NativeDecoderTest, UncompressedDictionaryRequiresEqualPhysicalAndL .is()); } +TEST(ParquetV2NativeDecoderTest, UncompressedDataPagesRequireEqualPhysicalAndLogicalSizes) { + for (const auto page_type : {tparquet::PageType::DATA_PAGE, tparquet::PageType::DATA_PAGE_V2}) { + tparquet::PageHeader header; + header.type = page_type; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(4); + if (page_type == tparquet::PageType::DATA_PAGE) { + header.__isset.data_page_header = true; + } else { + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + } + const std::vector payload(8); + EXPECT_TRUE(load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED) + .is()); + EXPECT_TRUE( + load_scripted_page(header, payload, tparquet::CompressionCodec::UNCOMPRESSED, true) + .is()); + } +} + +TEST(ParquetV2NativeDecoderTest, LegacyV2CompressedOverrideAllowsDifferentSizes) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__set_compressed_page_size(8); + header.__set_uncompressed_page_size(16); + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_is_compressed(false); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, true) + .ok()); + EXPECT_TRUE(validate_uncompressed_page_sizes(header, tparquet::CompressionCodec::SNAPPY, false) + .is()); +} + TEST(ParquetV2NativeDecoderTest, EmptyOffsetIndexCannotSelectIndexedPageReader) { MemoryBufferedReader reader(std::vector {0}); tparquet::ColumnMetaData metadata; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index c9239a152e5ab3..753e4516b38db8 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -1254,7 +1254,8 @@ class NewParquetReaderTest : public testing::Test { RuntimeProfile* profile = nullptr, bool enable_mapping_timestamp_tz = false, std::shared_ptr io_ctx = nullptr, std::optional global_rowid_context = std::nullopt, - bool is_immutable = false, bool enable_mapping_varbinary = false) const { + bool is_immutable = false, bool enable_mapping_varbinary = false, + std::string fs_name = {}, int64_t mtime = 0) const { auto system_properties = std::make_shared(); system_properties->system_type = TFileType::FILE_LOCAL; auto file_description = std::make_unique(); @@ -1263,6 +1264,8 @@ class NewParquetReaderTest : public testing::Test { file_description->range_start_offset = range_start_offset; file_description->range_size = range_size; file_description->is_immutable = is_immutable; + file_description->fs_name = std::move(fs_name); + file_description->mtime = mtime; return std::make_unique( system_properties, file_description, std::move(io_ctx), profile, global_rowid_context, enable_mapping_timestamp_tz, enable_mapping_varbinary); @@ -1952,6 +1955,74 @@ TEST_F(NewParquetReaderTest, UnknownMtimeSkipsPageCacheForMutableFile) { EXPECT_EQ(profile.get_counter("PageCacheWriteCount")->value(), 0); } +TEST_F(NewParquetReaderTest, NativeFooterCacheIdentityIncludesFilesystemAndVersion) { + const auto first = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto other_fs = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-b", "/warehouse/shared.parquet", 10, 0, 1024, 1024, false); + const auto replaced = format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/shared.parquet", 11, 0, 1024, 1024, false); + EXPECT_FALSE(first.empty()); + EXPECT_NE(first, other_fs); + EXPECT_NE(first, replaced); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotCrossFilesystems) { + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + RuntimeProfile first_profile("native_footer_cache_nameservice_a"); + auto first = create_reader(0, -1, &first_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-a", 1234567); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + RuntimeProfile second_profile("native_footer_cache_nameservice_b"); + auto second = create_reader(0, -1, &second_profile, false, nullptr, std::nullopt, false, false, + "hdfs://nameservice-b", 1234567); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheSkipsMutableUnknownVersion) { + EXPECT_TRUE(format::parquet::detail::build_native_file_cache_key("hdfs://nameservice-a", + "/warehouse/mutable.parquet", + 0, 0, 1024, 1024, false) + .empty()); + EXPECT_FALSE( + format::parquet::detail::build_native_file_cache_key( + "hdfs://nameservice-a", "/warehouse/immutable.parquet", 0, 0, 1024, 1024, true) + .empty()); +} + +TEST_F(NewParquetReaderTest, NativeFooterCacheDoesNotReuseMutableUnknownVersion) { + _file_path = (_test_dir / "mutable_footer_cache.parquet").string(); + write_parquet_file(_file_path); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + + RuntimeProfile first_profile("native_footer_cache_mutable_first"); + auto first = create_reader(0, -1, &first_profile); + ASSERT_TRUE(first->init(&state).ok()); + EXPECT_EQ(first_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(first_profile.get_counter("FileFooterHitCache")->value(), 0); + + write_parquet_file(_file_path); + RuntimeProfile second_profile("native_footer_cache_mutable_second"); + auto second = create_reader(0, -1, &second_profile); + ASSERT_TRUE(second->init(&state).ok()); + EXPECT_EQ(second_profile.get_counter("FileFooterReadCalls")->value(), 1); + EXPECT_EQ(second_profile.get_counter("FileFooterHitCache")->value(), 0); +} + +TEST_F(NewParquetReaderTest, NativeFooterSizeIsBoundedBeforeMetadataAllocation) { + constexpr size_t file_size = 256UL << 20; + constexpr size_t metadata_limit = 100UL << 20; + const auto status = format::parquet::detail::validate_native_footer_size( + static_cast(metadata_limit + 1), file_size, metadata_limit); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("metadata limit"), std::string::npos); +} + TEST_F(NewParquetReaderTest, UnknownMtimeUsesPageCacheForImmutableFile) { _file_path = (_test_dir / "unknown_mtime_page_cache.parquet").string(); write_parquet_file(_file_path); diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index d6d99230e8f474..83537dcc4e012e 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -47,6 +47,9 @@ #include "exprs/vslot_ref.h" #include "format_v2/file_reader.h" #include "format_v2/parquet/parquet_column_schema.h" +#include "format_v2/parquet/parquet_file_context.h" +#include "format_v2/parquet/reader/native/block_split_bloom_filter.h" +#include "io/fs/file_reader.h" #include "storage/index/bloom_filter/block_split_bloom_filter.h" #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" @@ -54,6 +57,37 @@ namespace doris { namespace { +class StatisticsMemoryFileReader final : public io::FileReader { +public: + explicit StatisticsMemoryFileReader(std::vector bytes) + : _bytes(std::move(bytes)), _path("native-bloom-filter.parquet") {} + + Status close() override { + _closed = true; + return Status::OK(); + } + const io::Path& path() const override { return _path; } + size_t size() const override { return _bytes.size(); } + bool closed() const override { return _closed; } + int64_t mtime() const override { return 1; } + +protected: + Status read_at_impl(size_t offset, Slice result, size_t* bytes_read, + const io::IOContext*) override { + if (offset > _bytes.size() || result.size > _bytes.size() - offset) { + return Status::IOError("native Bloom test read exceeds memory file"); + } + memcpy(result.data, _bytes.data() + offset, result.size); + *bytes_read = result.size; + return Status::OK(); + } + +private: + std::vector _bytes; + io::Path _path; + bool _closed = false; +}; + std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { std::shared_ptr array; EXPECT_TRUE(builder->Finish(&array).ok()); @@ -1067,6 +1101,99 @@ TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { *bloom_filter)); } +TEST(ParquetBloomFilterPruningTest, NativeUint32BloomUsesPhysicalInt32Hash) { + const auto column_schema = uint32_parquet_bloom_schema(); + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) + .ok()); + + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); + + EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field( + static_cast(present_value))}), + bloom_filter)); + EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::NativeBloomFilterExcludes( + column_schema, 0, + bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), + bloom_filter)); +} + +TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Max) { + auto column_schema = + std::make_unique(uint32_parquet_bloom_schema()); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + + format::parquet::native::BlockSplitBloomFilter bloom_filter; + ASSERT_TRUE(bloom_filter + .init(segment_v2::BloomFilter::MINIMUM_BYTES, + segment_v2::HashStrategyPB::XX_HASH_64) + .ok()); + const uint32_t present_value = 4000000000U; + int32_t physical_value; + memcpy(&physical_value, &present_value, sizeof(physical_value)); + bloom_filter.add_bytes(reinterpret_cast(&physical_value), sizeof(physical_value)); + + tparquet::BloomFilterAlgorithm algorithm; + algorithm.__set_BLOCK(tparquet::SplitBlockAlgorithm()); + tparquet::BloomFilterHash hash; + hash.__set_XXHASH(tparquet::XxHash()); + tparquet::BloomFilterCompression compression; + compression.__set_UNCOMPRESSED(tparquet::Uncompressed()); + tparquet::BloomFilterHeader bloom_header; + bloom_header.__set_numBytes(static_cast(bloom_filter.size())); + bloom_header.__set_algorithm(algorithm); + bloom_header.__set_hash(hash); + bloom_header.__set_compression(compression); + std::vector bloom_bytes; + ThriftSerializer serializer(/*compact=*/true, 64); + ASSERT_TRUE(serializer.serialize(&bloom_header, &bloom_bytes).ok()); + bloom_bytes.insert(bloom_bytes.end(), bloom_filter.data(), + bloom_filter.data() + bloom_filter.size()); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(1); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_bloom_filter_offset(0); + column_metadata.__set_bloom_filter_length(static_cast(bloom_bytes.size())); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_version(1); + metadata.__set_num_rows(1); + metadata.__set_row_groups({row_group}); + + format::parquet::ParquetFileContext file_context; + file_context.native_file = std::make_shared(std::move(bloom_bytes)); + auto request = request_with_bloom_conjunct( + column_schema->type, + {Field::create_field(static_cast(present_value))}); + std::vector> schema; + schema.push_back(std::move(column_schema)); + std::vector selected_row_groups; + format::parquet::ParquetPruningStats pruning_stats; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) + .ok()); + EXPECT_EQ(selected_row_groups, std::vector({0})); + EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); +} + TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { const auto column_schema = fixed_len_string_parquet_bloom_schema(4); auto bloom_filter = parquet_bloom_filter(); From 1001a585733bc79c4ae30962584f050c0cbbb99c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 14:53:22 +0800 Subject: [PATCH 20/34] [fix](be) Preserve Parquet geospatial byte arrays ### What problem does this PR solve? Issue Number: None Related PR: #65674 Problem Summary: The V2 native Parquet schema path accepts GEOMETRY and GEOGRAPHY BYTE_ARRAY leaves through physical type fallback, but reader initialization rejected both logical annotations. Permit only their valid BYTE_ARRAY pairing, preserve WKB as raw bytes, and keep invalid physical pairings corrupt. ### Release note None ### Check List (For Author) - Test: Unit Test - ParquetV2NativeDecoderTest.GeospatialByteArrayAnnotationsDecodeAsRawBytes - ParquetV2NativeDecoderTest.* - Behavior changed: Yes, valid Parquet geospatial BYTE_ARRAY leaves can be read as raw bytes - Does this need documentation: No --- .../parquet/reader/native/column_reader.cpp | 5 ++ .../format_v2/parquet/native_decoder_test.cpp | 60 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index d7fc92f2db9c98..c2b8c443372600 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -200,6 +200,11 @@ Status validate_physical_annotation(const NativeFieldSchema& field) { schema.type_length == 2, "FLOAT16"); } + if (logical.__isset.GEOMETRY || logical.__isset.GEOGRAPHY) { + // Geospatial WKB has no Doris logical mapping here; keep its BYTE_ARRAY payload raw so + // reader validation preserves the physical fallback chosen by native schema inference. + return require(field.physical_type == tparquet::Type::BYTE_ARRAY, "geospatial"); + } if (logical.__isset.UNKNOWN) { return Status::OK(); } diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 7634d714e8d457..1c8687a4606fa6 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -31,6 +31,7 @@ #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type_serde/parquet_timestamp.h" #include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" @@ -453,6 +454,65 @@ TEST(ParquetV2NativeDecoderTest, LegacyConvertedTimestampsRemainUtcAdjusted) { } } +TEST(ParquetV2NativeDecoderTest, GeospatialByteArrayAnnotationsDecodeAsRawBytes) { + const std::string wkb("\x01\x01\x00\x00\x00", 5); + for (const bool geometry : {true, false}) { + tparquet::LogicalType logical; + if (geometry) { + logical.__set_GEOMETRY(tparquet::GeometryType()); + } else { + logical.__set_GEOGRAPHY(tparquet::GeographyType()); + } + + NativeFieldSchema field; + field.name = geometry ? "geometry" : "geography"; + field.physical_type = tparquet::Type::BYTE_ARRAY; + field.parquet_schema.__set_logicalType(logical); + ParquetDecodeContext context; + ASSERT_TRUE(init_decode_context_for_test(field, nullptr, &context).ok()); + EXPECT_EQ(context.physical_type, ParquetPhysicalType::BYTE_ARRAY); + EXPECT_EQ(context.logical_type, ParquetLogicalType::NONE); + + DataTypeString type; + auto plain_column = type.create_column(); + auto encoded = encode_plain_byte_arrays({wkb}); + Slice plain_slice(encoded.data(), encoded.size()); + std::unique_ptr plain_decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, tparquet::Encoding::PLAIN, + plain_decoder) + .ok()); + ASSERT_TRUE(plain_decoder->set_data(&plain_slice).ok()); + ParquetMaterializationState plain_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*plain_column, *plain_decoder, context, 1, + plain_state) + .ok()); + ASSERT_EQ(plain_column->size(), 1); + EXPECT_EQ(plain_column->get_data_at(0).to_string_view(), wkb); + + int32_t dictionary_length = 0; + auto dictionary = make_byte_array_dictionary({wkb, "unused"}, &dictionary_length); + ByteArrayDictDecoder dictionary_decoder; + ASSERT_TRUE(dictionary_decoder.set_dict(dictionary, dictionary_length, 2).ok()); + char dictionary_index[] = {1, 2, 0}; + Slice dictionary_slice(dictionary_index, sizeof(dictionary_index)); + ASSERT_TRUE(dictionary_decoder.set_data(&dictionary_slice).ok()); + context.encoding = ParquetValueEncoding::DICTIONARY; + auto dictionary_column = type.create_column(); + ParquetMaterializationState dictionary_state; + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, dictionary_decoder, + context, 1, dictionary_state) + .ok()); + ASSERT_EQ(dictionary_column->size(), 1); + EXPECT_EQ(dictionary_column->get_data_at(0).to_string_view(), wkb); + + field.physical_type = tparquet::Type::INT32; + EXPECT_TRUE( + init_decode_context_for_test(field, nullptr, &context).is()); + } +} + TEST(ParquetV2NativeDecoderTest, InvalidLogicalPhysicalPairsFailBeforeDecode) { auto invalid = [](tparquet::Type::type physical, const tparquet::LogicalType& logical) { NativeFieldSchema field; From d682a26175b7287ef95d400b6ff6e84704304330 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 17:53:43 +0800 Subject: [PATCH 21/34] [refactor](be) remove Parquet V2 Arrow metadata adapters --- .../parquet/parquet_column_schema.cpp | 486 +------ .../format_v2/parquet/parquet_column_schema.h | 10 - .../parquet/parquet_file_context.cpp | 485 ------- .../format_v2/parquet/parquet_file_context.h | 69 - be/src/format_v2/parquet/parquet_reader.cpp | 68 +- be/src/format_v2/parquet/parquet_scan.cpp | 184 --- be/src/format_v2/parquet/parquet_scan.h | 16 - .../format_v2/parquet/parquet_statistics.cpp | 1247 +---------------- be/src/format_v2/parquet/parquet_statistics.h | 68 - be/src/format_v2/parquet/parquet_type.cpp | 300 +--- be/src/format_v2/parquet/parquet_type.h | 15 +- .../parquet/parquet_page_cache_range_test.cpp | 128 -- .../format_v2/parquet/parquet_reader_test.cpp | 315 ----- .../format_v2/parquet/parquet_scan_test.cpp | 237 ---- .../format_v2/parquet/parquet_schema_test.cpp | 504 ------- .../parquet/parquet_statistics_test.cpp | 1013 +------------ .../format_v2/parquet/parquet_type_test.cpp | 485 +------ 17 files changed, 107 insertions(+), 5523 deletions(-) diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index cefab47cf1cbc0..53b3bf189e0bc6 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -15,462 +15,18 @@ #include "format_v2/parquet/parquet_column_schema.h" -#include - #include #include #include #include -#include "core/data_type/data_type_array.h" -#include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_struct.h" #include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/parquet_type.h" namespace doris::format::parquet { namespace { -struct SchemaBuildContext { - int32_t local_id = -1; // child ordinal in the parent node - int16_t definition_level = 0; // accumulated optional/repeated level count - int16_t repetition_level = 0; // accumulated repeated level count - int16_t nullable_definition_level = 0; // definition level of the nearest optional node - int16_t repeated_repetition_level = 0; // repetition level of the nearest repeated node - int16_t repeated_ancestor_definition_level = 0; // definition level of the nearest repeated node -}; - -enum class SchemaBuildMode { - // Normal recursive schema build. Bare repeated fields are exposed as Doris ARRAY for - // protobuf/legacy Parquet compatibility, while repeated LIST/MAP annotated groups are rejected - // because Parquet LIST/MAP outer groups are not allowed to be repeated at a top-level or struct - // field boundary. - NORMAL, - // Build the current repeated node as the already-selected element of an enclosing LIST. This - // is the compatibility path for Arrow/parquet-format legacy two-level LIST encodings where the - // repeated node itself is the array element instead of a wrapper that should be stripped. - REPEATED_NODE_AS_LIST_ELEMENT, - // Build the current repeated group as a STRUCT element of an enclosing LIST, ignoring LIST/MAP - // annotations on the repeated group itself. This keeps compatibility with the old Doris - // Parquet schema parser for Hive/legacy wrappers named "array" or "_tuple". - REPEATED_NODE_AS_STRUCT_ELEMENT, -}; - -// Result of applying Parquet LIST backward compatibility rules to the single repeated child of a -// LIST-annotated group. The repeated child can either be a physical wrapper whose only child is the -// element, or the element node itself. -struct ListElementResolution { - // Parquet node that should be exposed as Doris ARRAY element. - const ::parquet::schema::Node* element_node = nullptr; - // Level state after consuming the LIST repeated child. The parent ARRAY schema keeps this state - // to materialize offsets, empty arrays and null arrays. - SchemaBuildContext repeated_context; - // Level state used to build element_node. This equals repeated_context when the repeated child - // itself is the element, and includes the wrapper's only child when standard 3-level LIST - // encoding is stripped. - SchemaBuildContext element_context; - // Build mode for element_node. Non-NORMAL modes mean element_node is the repeated child itself, - // and the repeated level must not be interpreted as a second unrelated array at the same - // boundary. - SchemaBuildMode element_build_mode = SchemaBuildMode::NORMAL; -}; - -// Resolved repeated entry group of a MAP-annotated group. The entry wrapper is a physical Parquet -// encoding detail; Doris folds it into the parent MAP schema and exposes only direct [key, value] -// children. -struct MapEntryResolution { - const ::parquet::schema::GroupNode* entry_group = nullptr; - // Level state after consuming the repeated entry group. The parent MAP schema keeps this state - // to materialize offsets, empty maps and null maps. - SchemaBuildContext entry_context; -}; - -bool is_list_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::LIST || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_list()); -} - -bool is_map_node(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return node.converted_type() == ::parquet::ConvertedType::MAP || - node.converted_type() == ::parquet::ConvertedType::MAP_KEY_VALUE || - (logical_type != nullptr && logical_type->is_valid() && logical_type->is_map()); -} - -bool has_logical_annotation(const ::parquet::schema::Node& node) { - const auto& logical_type = node.logical_type(); - return (node.converted_type() != ::parquet::ConvertedType::NONE && - node.converted_type() != ::parquet::ConvertedType::UNDEFINED) || - (logical_type != nullptr && logical_type->is_valid() && !logical_type->is_none()); -} - -bool has_structural_list_name(const std::string& list_name, const std::string& repeated_name) { - return repeated_name == "array" || repeated_name == list_name + "_tuple"; -} - -bool should_build_repeated_field_as_list(const ::parquet::schema::Node& node) { - return node.is_repeated() && !is_list_node(node) && !is_map_node(node); -} - -DataTypePtr nullable_if_needed(DataTypePtr type, const ::parquet::schema::Node& node) { - return node.is_optional() ? make_nullable(type) : type; -} - -void inherit_common_schema_state(const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - column_schema->local_id = context.local_id; - column_schema->parquet_field_id = node.field_id(); - column_schema->name = node.name(); - column_schema->max_definition_level = context.definition_level; - column_schema->max_repetition_level = context.repetition_level; - column_schema->nullable_definition_level = context.nullable_definition_level; - column_schema->definition_level = context.definition_level; - column_schema->repetition_level = context.repetition_level; - column_schema->repeated_ancestor_definition_level = context.repeated_ancestor_definition_level; - column_schema->repeated_repetition_level = context.repeated_repetition_level; -} - -SchemaBuildContext child_context(const SchemaBuildContext& parent, - const ::parquet::schema::Node& child_node, int32_t child_idx) { - SchemaBuildContext result = parent; - result.local_id = child_idx; - if (child_node.repetition() == ::parquet::Repetition::OPTIONAL) { - result.definition_level++; - result.nullable_definition_level = result.definition_level; - } - if (child_node.is_repeated()) { - result.repetition_level++; - result.definition_level++; - result.repeated_repetition_level = result.repetition_level; - result.repeated_ancestor_definition_level = result.definition_level; - } - return result; -} - -void propagate_child_levels(ParquetColumnSchema* column_schema) { - DORIS_CHECK(column_schema != nullptr); - for (const auto& child : column_schema->children) { - column_schema->max_definition_level = - std::max(column_schema->max_definition_level, child->max_definition_level); - column_schema->max_repetition_level = - std::max(column_schema->max_repetition_level, child->max_repetition_level); - } -} - -// Mirrors Arrow's ResolveList() compatibility rules, but only decides which Parquet node is the -// logical LIST element. The caller still builds Doris' semantic LIST->[element] schema tree. -// Important cases: -// - repeated primitive: the primitive itself is the element (legacy two-level LIST). -// - repeated group with multiple children: the group itself is a STRUCT element. -// - repeated group named "array" or "_tuple": the group itself is a STRUCT element per -// Parquet backward compatibility rules, even when it has one child or its own logical annotation. -// This also keeps v2 file-local schema aligned with Doris' old schema parser used by HDFS TVF. -// - other repeated group with a logical annotation, or whose only child is repeated: the group -// itself is the element. This preserves nested LIST/MAP and repeated fields inside struct -// elements. -// - otherwise, strip the one-child repeated wrapper as standard three-level LIST encoding. -Status resolve_list_element_node(const ::parquet::schema::GroupNode& list_group, - const SchemaBuildContext& list_context, - ListElementResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (list_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - const auto& repeated_node = *list_group.field(0); - if (!repeated_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet LIST encoding for column {}", - list_group.name()); - } - result->repeated_context = child_context(list_context, repeated_node, 0); - if (repeated_node.is_primitive()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& repeated_group = static_cast(repeated_node); - if (repeated_group.field_count() == 0) { - return Status::NotSupported("Unsupported parquet LIST element layout for column {}", - list_group.name()); - } - const bool repeated_group_has_logical_annotation = has_logical_annotation(repeated_group); - if (repeated_group.field_count() > 1 || - has_structural_list_name(list_group.name(), repeated_group.name())) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT; - return Status::OK(); - } - if (repeated_group_has_logical_annotation) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - const auto& only_child = *repeated_group.field(0); - if (only_child.is_repeated()) { - result->element_node = &repeated_node; - result->element_context = result->repeated_context; - result->element_build_mode = SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT; - return Status::OK(); - } - - result->element_node = &only_child; - result->element_context = child_context(result->repeated_context, only_child, 0); - return Status::OK(); -} - -// Resolves the repeated entry group of a MAP/MAP_KEY_VALUE node. Unlike LIST, MAP has no supported -// two-level form in this reader: Doris requires a repeated group with exactly key and value -// children, then folds that physical entry group out of ParquetColumnSchema. Some external writers -// emit optional MAP keys even though standard Parquet MAP keys are required; keep the key's -// definition levels and expose it as nullable for compatibility with the old reader. -Status resolve_map_entry_group(const ::parquet::schema::GroupNode& map_group, - const SchemaBuildContext& map_context, MapEntryResolution* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (map_group.field_count() != 1) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - const auto& entry_node = *map_group.field(0); - if (!entry_node.is_repeated()) { - return Status::NotSupported("Unsupported parquet MAP encoding for column {}", - map_group.name()); - } - if (entry_node.is_primitive()) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - const auto& entry_group = static_cast(entry_node); - if (entry_group.field_count() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - map_group.name()); - } - // The Parquet logical MAP spec requires key to be REQUIRED. Some legacy/Hive-written files - // still mark the key field OPTIONAL even when all actual keys are non-null, for example: - // optional group t_map_varchar (MAP) { - // repeated group key_value { - // optional binary key (STRING); - // optional binary value (STRING); - // } - // } - // Accept that schema here so compatible files can be read. The native reader validates the - // materialized key column and rejects data that really contains null map keys. - result->entry_group = &entry_group; - result->entry_context = child_context(map_context, entry_node, 0); - return Status::OK(); -} - -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode); - -// Builds a semantic ARRAY schema for a bare repeated field. Arrow handles this in -// NodeToSchemaField()/GroupToSchemaField(); Doris needs the same compatibility behavior because -// protobuf and old parquet writers often encode repeated fields without a LIST annotation. -// Example: -// optional group event { -// repeated group links { -// optional binary url (UTF8); -// optional int32 rank; -// } -// } -// Doris exposes event.links as ARRAY>, not STRUCT. This keeps v2's -// file-local schema aligned with the old schema parser used by HDFS TVF schema fetching. -// When the repeated field appears inside an already resolved LIST element, only the nested repeated -// child should be wrapped: -// optional group a (LIST) { -// repeated group element { -// repeated int32 items; -// } -// } -// The outer LIST element is the repeated "element" group, and its repeated "items" child should be -// represented as a field of type ARRAY inside the struct element. -Status build_repeated_field_as_list_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& repeated_node, - const SchemaBuildContext& repeated_context, - std::unique_ptr* result) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - auto list_schema = std::make_unique(); - inherit_common_schema_state(repeated_node, repeated_context, list_schema.get()); - list_schema->kind = ParquetColumnSchemaKind::LIST; - list_schema->definition_level = repeated_context.definition_level; - list_schema->repetition_level = repeated_context.repetition_level; - list_schema->repeated_repetition_level = repeated_context.repeated_repetition_level; - - std::unique_ptr element_child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, repeated_node, repeated_context, - &element_child, - SchemaBuildMode::REPEATED_NODE_AS_LIST_ELEMENT)); - element_child->name = "element"; - list_schema->type = std::make_shared(element_child->type); - list_schema->children.push_back(std::move(element_child)); - propagate_child_levels(list_schema.get()); - *result = std::move(list_schema); - return Status::OK(); -} - -// Recursively builds ParquetColumnSchema for the given schema node and its children in Parquet -// file's metadata. NORMAL mode exposes bare repeated fields as ARRAY for legacy compatibility. -// REPEATED_NODE_AS_LIST_ELEMENT mode means the current repeated node was already selected as an -// enclosing LIST element, so only its nested bare repeated children should be wrapped. -Status build_node_schema_with_mode(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, - const SchemaBuildContext& context, - std::unique_ptr* result, - SchemaBuildMode mode) { - if (result == nullptr) { - return Status::InvalidArgument("result is null"); - } - if (mode == SchemaBuildMode::NORMAL && should_build_repeated_field_as_list(node)) { - return build_repeated_field_as_list_schema(schema, node, context, result); - } - - auto column_schema = std::make_unique(); - inherit_common_schema_state(node, context, column_schema.get()); - - if (node.is_primitive()) { - const int leaf_column_id = schema.ColumnIndex(node); - if (leaf_column_id < 0) { - return Status::InvalidArgument("Cannot find leaf column id for parquet column {}", - node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::PRIMITIVE; - column_schema->leaf_column_id = leaf_column_id; - column_schema->descriptor = schema.Column(leaf_column_id); - if (column_schema->descriptor != nullptr) { - column_schema->max_definition_level = column_schema->descriptor->max_definition_level(); - column_schema->max_repetition_level = column_schema->descriptor->max_repetition_level(); - } - column_schema->type_descriptor = resolve_parquet_type(column_schema->descriptor); - column_schema->type = column_schema->type_descriptor.doris_type; - if (column_schema->type == nullptr && - !column_schema->type_descriptor.unsupported_reason.empty()) { - // Keep unsupported logical leaves in the file schema using their physical storage - // type. For example, a file `{id: INT32, clock: TIME_MILLIS}` remains readable for - // `SELECT id`: schema mapping sees `clock` as its physical INT32 but never creates its - // reader. `SELECT clock` still fails explicitly in ParquetColumnReaderFactory before - // any physical value is decoded, preserving the unsupported-type contract. - column_schema->type = column_schema->type_descriptor.physical_doris_type; - } - if (column_schema->type == nullptr) { - return Status::NotSupported("Unsupported parquet column type for column {}", - node.name()); - } - column_schema->type = node.is_optional() - ? make_nullable(remove_nullable(column_schema->type)) - : remove_nullable(column_schema->type); - *result = std::move(column_schema); - return Status::OK(); - } - - const auto& group = static_cast(node); - if (is_list_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet LIST column {}", node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::LIST; - ListElementResolution list_element; - RETURN_IF_ERROR(resolve_list_element_node(group, context, &list_element)); - column_schema->definition_level = list_element.repeated_context.definition_level; - column_schema->repetition_level = list_element.repeated_context.repetition_level; - column_schema->repeated_repetition_level = - list_element.repeated_context.repeated_repetition_level; - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode(schema, *list_element.element_node, - list_element.element_context, &child, - list_element.element_build_mode)); - child->name = "element"; - column_schema->type = - nullable_if_needed(std::make_shared(child->type), node); - column_schema->children.push_back(std::move(child)); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); - } - - if (is_map_node(node) && mode != SchemaBuildMode::REPEATED_NODE_AS_STRUCT_ELEMENT) { - if (mode == SchemaBuildMode::NORMAL && node.is_repeated()) { - return Status::NotSupported("Unsupported repeated parquet MAP column {}", node.name()); - } - column_schema->kind = ParquetColumnSchemaKind::MAP; - MapEntryResolution map_entry; - RETURN_IF_ERROR(resolve_map_entry_group(group, context, &map_entry)); - column_schema->definition_level = map_entry.entry_context.definition_level; - column_schema->repetition_level = map_entry.entry_context.repetition_level; - column_schema->repeated_repetition_level = - map_entry.entry_context.repeated_repetition_level; - for (int child_idx = 0; child_idx < map_entry.entry_group->field_count(); ++child_idx) { - std::unique_ptr child; - RETURN_IF_ERROR(build_node_schema_with_mode( - schema, *map_entry.entry_group->field(child_idx), - child_context(map_entry.entry_context, *map_entry.entry_group->field(child_idx), - child_idx), - &child, SchemaBuildMode::NORMAL)); - child->name = child_idx == 0 ? "key" : "value"; - column_schema->children.push_back(std::move(child)); - } - if (column_schema->children.size() != 2) { - return Status::NotSupported("Unsupported parquet MAP key_value layout for column {}", - node.name()); - } - auto key_type = make_nullable(column_schema->children[0]->type); - auto value_type = make_nullable(column_schema->children[1]->type); - column_schema->type = - nullable_if_needed(std::make_shared(key_type, value_type), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); - } - - column_schema->kind = ParquetColumnSchemaKind::STRUCT; - DataTypes child_types; - Strings child_names; - child_types.reserve(group.field_count()); - child_names.reserve(group.field_count()); - for (int child_idx = 0; child_idx < group.field_count(); ++child_idx) { - const auto& child_node = *group.field(child_idx); - std::unique_ptr child; - const auto child_ctx = child_context(context, child_node, child_idx); - if (should_build_repeated_field_as_list(child_node)) { - RETURN_IF_ERROR( - build_repeated_field_as_list_schema(schema, child_node, child_ctx, &child)); - } else { - RETURN_IF_ERROR(build_node_schema_with_mode(schema, child_node, child_ctx, &child, - SchemaBuildMode::NORMAL)); - } - child_types.push_back(make_nullable(child->type)); - child_names.push_back(child->name); - column_schema->children.push_back(std::move(child)); - } - column_schema->type = - nullable_if_needed(std::make_shared(child_types, child_names), node); - propagate_child_levels(column_schema.get()); - *result = std::move(column_schema); - return Status::OK(); -} - -Status build_node_schema(const ::parquet::SchemaDescriptor& schema, - const ::parquet::schema::Node& node, const SchemaBuildContext& context, - std::unique_ptr* result) { - return build_node_schema_with_mode(schema, node, context, result, SchemaBuildMode::NORMAL); -} - ParquetTimeUnit native_time_unit(const tparquet::TimeUnit& unit) { if (unit.__isset.MILLIS) { return ParquetTimeUnit::MILLIS; @@ -502,12 +58,8 @@ void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDesc DORIS_CHECK(result != nullptr); const auto& schema = field.parquet_schema; result->doris_type = field.data_type; - result->physical_type = static_cast<::parquet::Type::type>(field.physical_type); + result->physical_type = static_cast(field.physical_type); result->fixed_length = schema.__isset.type_length ? schema.type_length : -1; - if (schema.__isset.converted_type) { - result->converted_type = static_cast<::parquet::ConvertedType::type>(schema.converted_type); - } - if (schema.__isset.logicalType) { const auto& logical = schema.logicalType; if (logical.__isset.DECIMAL) { @@ -579,27 +131,27 @@ void fill_native_type_descriptor(const NativeFieldSchema& field, ParquetTypeDesc if (result->is_decimal) { switch (result->physical_type) { - case ::parquet::Type::INT32: + case tparquet::Type::INT32: result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; break; - case ::parquet::Type::INT64: + case tparquet::Type::INT64: result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; break; default: break; } - } else if (result->physical_type == ::parquet::Type::INT96) { + } else if (result->physical_type == tparquet::Type::INT96) { result->is_timestamp = true; result->extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; } result->is_string_like = !result->is_decimal && result->extra_type_info != ParquetExtraTypeInfo::FLOAT16 && - (result->physical_type == ::parquet::Type::BYTE_ARRAY || - result->physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); + (result->physical_type == tparquet::Type::BYTE_ARRAY || + result->physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY); } void propagate_native_max_levels(ParquetColumnSchema* schema) { @@ -674,28 +226,6 @@ Status validate_native_node_schema(const NativeFieldSchema& field) { } // namespace -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, - std::vector>* fields) { - if (fields == nullptr) { - return Status::InvalidArgument("fields is null"); - } - fields->clear(); - const auto* root = schema.group_node(); - if (root == nullptr) { - return Status::InvalidArgument("Parquet schema root is null"); - } - fields->reserve(root->field_count()); - for (int field_idx = 0; field_idx < root->field_count(); ++field_idx) { - std::unique_ptr field; - SchemaBuildContext context; - RETURN_IF_ERROR(build_node_schema( - schema, *root->field(field_idx), - child_context(context, *root->field(field_idx), field_idx), &field)); - fields->push_back(std::move(field)); - } - return Status::OK(); -} - Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields) { if (fields == nullptr) { diff --git a/be/src/format_v2/parquet/parquet_column_schema.h b/be/src/format_v2/parquet/parquet_column_schema.h index 76ae58b24d2c0a..697bcd498382b1 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.h +++ b/be/src/format_v2/parquet/parquet_column_schema.h @@ -23,11 +23,6 @@ #include "core/data_type/data_type.h" #include "format_v2/parquet/parquet_type.h" -namespace parquet { -class ColumnDescriptor; -class SchemaDescriptor; -} // namespace parquet - namespace doris::format::parquet { class NativeFieldDescriptor; @@ -57,8 +52,6 @@ struct ParquetColumnSchema { ParquetColumnSchemaKind kind = ParquetColumnSchemaKind::PRIMITIVE; - const ::parquet::ColumnDescriptor* descriptor = nullptr; - // ======== Dremel Levels ======== int16_t max_definition_level = 0; @@ -76,9 +69,6 @@ struct ParquetColumnSchema { std::vector> children {}; }; -Status build_parquet_column_schema(const ::parquet::SchemaDescriptor& schema, - std::vector>* fields); - Status build_parquet_column_schema(const NativeFieldDescriptor& schema, std::vector>* fields); diff --git a/be/src/format_v2/parquet/parquet_file_context.cpp b/be/src/format_v2/parquet/parquet_file_context.cpp index 1dc545afa3934d..00956b3d09ce65 100644 --- a/be/src/format_v2/parquet/parquet_file_context.cpp +++ b/be/src/format_v2/parquet/parquet_file_context.cpp @@ -15,19 +15,12 @@ #include "format_v2/parquet/parquet_file_context.h" -#include -#include #include -#include -#include -#include -#include #include #include #include #include -#include #include #include @@ -44,7 +37,6 @@ #include "io/fs/tracing_file_reader.h" #include "io/io_common.h" #include "runtime/exec_env.h" -#include "storage/cache/page_cache.h" #include "util/coding.h" #include "util/slice.h" #include "util/thrift_util.h" @@ -103,75 +95,6 @@ Status NativeParquetMetadata::init_schema(bool enable_mapping_varbinary, namespace detail { -namespace { - -bool page_cache_range_less(const ParquetPageCacheRange& lhs, const ParquetPageCacheRange& rhs) { - return lhs.offset < rhs.offset || (lhs.offset == rhs.offset && lhs.size < rhs.size); -} - -} // namespace - -ParquetPageCacheRangeIndex::ParquetPageCacheRangeIndex(size_t max_ranges) - : _max_ranges(max_ranges) { - DORIS_CHECK(_max_ranges > 0); -} - -void ParquetPageCacheRangeIndex::insert(ParquetPageCacheRange range) { - std::lock_guard lock(_mutex); - auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { - return; - } - if (_ranges.size() >= _max_ranges) { - _ranges.erase(_ranges.begin()); - it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - } - _ranges.insert(it, range); -} - -void ParquetPageCacheRangeIndex::erase(ParquetPageCacheRange range) { - std::lock_guard lock(_mutex); - const auto it = std::lower_bound(_ranges.begin(), _ranges.end(), range, page_cache_range_less); - if (it != _ranges.end() && it->offset == range.offset && it->size == range.size) { - _ranges.erase(it); - } -} - -std::vector ParquetPageCacheRangeIndex::ranges() const { - std::lock_guard lock(_mutex); - return _ranges; -} - -size_t ParquetPageCacheRangeIndex::size() const { - std::lock_guard lock(_mutex); - return _ranges.size(); -} - -ParquetPageCacheRangeDirectory::ParquetPageCacheRangeDirectory(size_t max_files) - : _max_files(max_files) { - DORIS_CHECK(_max_files > 0); -} - -std::shared_ptr ParquetPageCacheRangeDirectory::get_or_create( - const std::string& file_key) { - DORIS_CHECK(!file_key.empty()); - std::lock_guard lock(_mutex); - if (const auto it = _indexes.find(file_key); it != _indexes.end()) { - return it->second; - } - if (_indexes.size() >= _max_files) { - _indexes.erase(_indexes.begin()); - } - auto index = std::make_shared(); - _indexes.emplace(file_key, index); - return index; -} - -size_t ParquetPageCacheRangeDirectory::size() const { - std::lock_guard lock(_mutex); - return _indexes.size(); -} - Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, size_t metadata_size_limit) { if (file_size < V2_PARQUET_FOOTER_SIZE || @@ -213,58 +136,6 @@ bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end) { span_end - span_offset <= MAX_SERIALIZED_PARQUET_INDEX_BYTES; } -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges) { - if (position < 0 || nbytes <= 0) { - return {}; - } - - std::vector ranges; - ranges.reserve(cached_ranges.size()); - const int64_t request_end = position + nbytes; - for (const auto& range : cached_ranges) { - if (range.size > 0 && range.offset < request_end && position < range.end_offset()) { - ranges.push_back(range); - } - } - std::sort(ranges.begin(), ranges.end(), [](const auto& lhs, const auto& rhs) { - if (lhs.offset != rhs.offset) { - return lhs.offset < rhs.offset; - } - return lhs.size > rhs.size; - }); - - std::vector plan; - int64_t cursor = position; - while (cursor < request_end) { - // At each cursor position, choose the cached range that already covers the cursor and - // extends farthest to the right. This handles both adjacent ranges and overlapping - // ranges. If no range covers the current cursor, there is a gap and the request must - // miss as a whole. - auto best = ranges.end(); - int64_t best_end = cursor; - for (auto it = ranges.begin(); it != ranges.end(); ++it) { - const int64_t cached_end = it->end_offset(); - if (it->offset <= cursor && cursor < cached_end && cached_end > best_end) { - best = it; - best_end = cached_end; - } - } - if (best == ranges.end()) { - return {}; - } - const int64_t copy_size = std::min(best_end, request_end) - cursor; - ParquetPageCacheReadPlanEntry entry; - entry.cached_range = *best; - entry.copy_offset_in_cache = cursor - best->offset; - entry.output_offset = cursor - position; - entry.copy_size = copy_size; - plan.push_back(entry); - cursor += copy_size; - } - return plan; -} - std::vector valid_prefetch_ranges( const std::vector& ranges) { std::vector valid_ranges; @@ -307,16 +178,6 @@ bool should_stage_small_http_file(std::string_view path, size_t file_size, namespace { -detail::ParquetPageCacheRangeDirectory& cached_page_range_directory() { - // Directory lookup is paid once when a reader opens. ReadAt() then synchronizes only on the - // shared index for this file, so unrelated Parquet files no longer serialize on a process-wide - // hot-path mutex. Strong references deliberately keep range discovery alive after reader A - // closes: reader B can reuse cached [100, 200) for a request [120, 150). The directory and each - // per-file index are independently capped, bounding stale metadata left by page-cache eviction. - static detail::ParquetPageCacheRangeDirectory directory; - return directory; -} - constexpr uint8_t V2_PARQUET_MAGIC[4] = {'P', 'A', 'R', '1'}; constexpr size_t V2_INITIAL_FOOTER_READ_SIZE = 48 * 1024; @@ -392,344 +253,8 @@ std::string build_page_cache_file_key(const io::FileReader& file_reader, static_cast(file_reader.size()), file_description.is_immutable); } -class DorisRandomAccessFile final : public arrow::io::RandomAccessFile { -public: - [[maybe_unused]] DorisRandomAccessFile(io::FileReaderSPtr file_reader, io::IOContext* io_ctx, - bool enable_page_cache, std::string page_cache_file_key) - : _file_reader(std::move(file_reader)), - _base_file_reader(_file_reader), - _io_ctx(io_ctx), - _enable_page_cache(enable_page_cache), - _page_cache_file_key(std::move(page_cache_file_key)), - _cached_page_range_index( - _enable_page_cache && !_page_cache_file_key.empty() - ? cached_page_range_directory().get_or_create(_page_cache_file_key) - : nullptr) { - DORIS_CHECK(_file_reader != nullptr); - if (auto tracing_reader = std::dynamic_pointer_cast(_file_reader)) { - _file_reader_stats = tracing_reader->stats(); - _base_file_reader = tracing_reader->inner_reader(); - } - DORIS_CHECK(_base_file_reader != nullptr); - set_mode(arrow::io::FileMode::READ); - } - - arrow::Status Close() override { - if (!_closed) { - collect_active_merge_range_profile(); - std::lock_guard lock(_page_cache_mutex); - // Page payloads and their bounded per-file range index intentionally outlive this - // reader for warm scans. Only reader-specific projected ranges are released here. - std::vector().swap(_page_cache_ranges); - _closed = true; - } - return arrow::Status::OK(); - } - - bool closed() const override { return _closed; } - - arrow::Result Tell() const override { return _pos; } - - arrow::Status Seek(int64_t position) override { - if (position < 0) { - return arrow::Status::Invalid("negative seek position"); - } - _pos = position; - return arrow::Status::OK(); - } - - arrow::Result GetSize() override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - return static_cast(_file_reader->size()); - } - - arrow::Result Read(int64_t nbytes, void* out) override { - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(_pos, nbytes, out)); - _pos += bytes_read; - return bytes_read; - } - - arrow::Result> Read(int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, Read(nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; - } - - arrow::Result ReadAt(int64_t position, int64_t nbytes, void* out) override { - if (!_file_reader) { - return arrow::Status::IOError("Doris file reader is not open"); - } - if (_io_ctx != nullptr && _io_ctx->should_stop) { - return arrow::Status::IOError("stop"); - } - if (position < 0 || nbytes < 0) { - return arrow::Status::Invalid("negative read position or length"); - } - if (try_read_from_page_cache(position, nbytes, out)) { - return nbytes; - } - size_t bytes_read = 0; - Status st = _file_reader->read_at( - static_cast(position), - Slice(static_cast(out), static_cast(nbytes)), &bytes_read, - _io_ctx); - if (!st.ok()) { - return arrow::Status::IOError(st.to_string_no_stack()); - } - insert_page_cache(position, nbytes, out, bytes_read); - return static_cast(bytes_read); - } - - arrow::Result> ReadAt(int64_t position, - int64_t nbytes) override { - ARROW_ASSIGN_OR_RAISE(auto buffer, arrow::AllocateResizableBuffer(nbytes)); - ARROW_ASSIGN_OR_RAISE(auto bytes_read, ReadAt(position, nbytes, buffer->mutable_data())); - ARROW_RETURN_NOT_OK(buffer->Resize(bytes_read, false)); - buffer->ZeroPadding(); - return buffer; - } - - [[maybe_unused]] void register_page_cache_ranges(std::vector ranges) { - std::lock_guard lock(_page_cache_mutex); - _page_cache_ranges = std::move(ranges); - } - - [[maybe_unused]] void prefetch_ranges(const std::vector& ranges, - const io::IOContext* io_ctx) { - auto cached_reader = cached_remote_file_reader(); - if (cached_reader == nullptr) { - return; - } - const auto* prefetch_io_ctx = io_ctx != nullptr ? io_ctx : _io_ctx; - for (const auto& range : ranges) { - if (range.offset < 0 || range.size <= 0) { - continue; - } - cached_reader->prefetch_range(static_cast(range.offset), - static_cast(range.size), prefetch_io_ctx); - } - } - - [[maybe_unused]] bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - reset_active_file_reader(); - const auto valid_ranges = detail::valid_prefetch_ranges(ranges); - if (!detail::should_use_merge_range_reader( - valid_ranges, avg_io_size, - typeid_cast(_base_file_reader.get()) != nullptr)) { - return false; - } - - std::vector random_access_ranges; - random_access_ranges.reserve(valid_ranges.size()); - for (const auto& range : valid_ranges) { - random_access_ranges.emplace_back(static_cast(range.offset), - static_cast(range.end_offset())); - } - - // This mirrors the v1 parquet reader for the migration metadata/index ReadAt path. Native - // data-page decoding owns a separate BufferedFileStreamReader and v1-compatible page cache; - // adjacent metadata/index requests here can still be coalesced and served from merge - // buffers. - // Example: a row group projects leaf chunks [1MB, 1.5MB) and [1.6MB, 2MB). Arrow later - // issues page reads inside those chunks; MergeRangeFileReader can fetch a wider slice once - // and satisfy the following ReadAt calls from its boxes, reducing remote request count. - _merge_range_active = true; - set_active_file_reader(std::make_shared( - profile, _base_file_reader, random_access_ranges, merge_read_slice_size)); - return true; - } - - [[maybe_unused]] void reset_random_access_ranges() { reset_active_file_reader(); } - - [[maybe_unused]] ParquetPageCacheStats page_cache_stats() const { - std::lock_guard lock(_page_cache_mutex); - return _page_cache_stats; - } - -private: - bool page_cache_enabled() const { - return _enable_page_cache && !config::disable_storage_page_cache && - StoragePageCache::instance() != nullptr && !_page_cache_file_key.empty() && - _cached_page_range_index != nullptr; - } - - bool range_in_page_cache_scope(int64_t position, int64_t nbytes) const { - if (nbytes <= 0) { - return false; - } - const int64_t end = position + nbytes; - for (const auto& range : _page_cache_ranges) { - const int64_t range_end = range.offset + range.size; - if (position >= range.offset && end <= range_end) { - return true; - } - } - return false; - } - - StoragePageCache::CacheKey page_cache_key(int64_t position, int64_t nbytes) const { - return StoragePageCache::CacheKey(_page_cache_file_key, - static_cast(position + nbytes), position); - } - - bool copy_cached_range(const ParquetPageCacheRange& cached_range, int64_t copy_position, - int64_t copy_size, void* out, int64_t output_offset) { - PageCacheHandle handle; - if (!StoragePageCache::instance()->lookup( - page_cache_key(cached_range.offset, cached_range.size), &handle, - segment_v2::DATA_PAGE)) { - _cached_page_range_index->erase(cached_range); - return false; - } - Slice cached = handle.data(); - const int64_t cache_offset = copy_position - cached_range.offset; - DORIS_CHECK(cache_offset >= 0); - DORIS_CHECK(cached.size >= static_cast(cache_offset + copy_size)); - memcpy(static_cast(out) + output_offset, cached.data + cache_offset, - static_cast(copy_size)); - return true; - } - - bool try_read_from_cached_ranges(int64_t position, int64_t nbytes, void* out) { - auto plan = detail::plan_page_cache_range_read(position, nbytes, - _cached_page_range_index->ranges()); - if (plan.empty()) { - return false; - } - for (const auto& entry : plan) { - if (!copy_cached_range(entry.cached_range, - entry.cached_range.offset + entry.copy_offset_in_cache, - entry.copy_size, out, entry.output_offset)) { - return false; - } - } - return true; - } - - bool try_read_from_page_cache(int64_t position, int64_t nbytes, void* out) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes)) { - return false; - } - ++_page_cache_stats.read_count; - // Fast path: Arrow issues the same ReadAt(offset, size) again, so the exact - // StoragePageCache key matches. - // Fallback path: Arrow may read a different but related byte range on another scan. - // Examples: - // - Current request [120, 150) can be served from cached [100, 200) by copying the - // 30-byte subset starting at cached offset 20. - // - Current request [100, 260) can be served by stitching cached [100, 180) and - // [180, 260). If any middle span is missing, it is a miss and the file reader fills - // the whole request from storage. - if (!copy_cached_range(ParquetPageCacheRange {position, nbytes}, position, nbytes, out, - 0) && - !try_read_from_cached_ranges(position, nbytes, out)) { - ++_page_cache_stats.miss_count; - return false; - } - ++_page_cache_stats.hit_count; - ++_page_cache_stats.compressed_hit_count; - return true; - } - - void insert_page_cache(int64_t position, int64_t nbytes, const void* data, size_t bytes_read) { - std::lock_guard lock(_page_cache_mutex); - if (!page_cache_enabled() || !range_in_page_cache_scope(position, nbytes) || - bytes_read != static_cast(nbytes)) { - return; - } - auto* page = new DataPage(bytes_read, true, segment_v2::DATA_PAGE); - memcpy(page->data(), data, bytes_read); - PageCacheHandle handle; - StoragePageCache::instance()->insert(page_cache_key(position, nbytes), page, &handle, - segment_v2::DATA_PAGE); - _cached_page_range_index->insert( - ParquetPageCacheRange {.offset = position, .size = nbytes}); - ++_page_cache_stats.write_count; - ++_page_cache_stats.compressed_write_count; - } - - void set_active_file_reader(io::FileReaderSPtr reader) { - DORIS_CHECK(reader != nullptr); - _file_reader = _file_reader_stats != nullptr - ? std::make_shared(std::move(reader), - _file_reader_stats) - : std::move(reader); - } - - void reset_active_file_reader() { - collect_active_merge_range_profile(); - _merge_range_active = false; - set_active_file_reader(_base_file_reader); - } - - void collect_active_merge_range_profile() { - if (_merge_range_active && _file_reader != nullptr) { - // MergeRangeFileReader writes its MergedSmallIO counters only from - // collect_profile_before_close(). v2 replaces the active reader for every row group, - // so collect before overwriting it; Close() handles the final row group. Example: - // RG0 installs a merge reader, RG1 calls set_random_access_ranges() and resets the - // active reader first, so RG0's RequestIO/MergedIO counters must be flushed here. - _file_reader->collect_profile_before_close(); - } - } - - std::shared_ptr cached_remote_file_reader() { - if (_merge_range_active) { - return nullptr; - } - auto reader = _file_reader; - if (reader == nullptr) { - return nullptr; - } - // FileReader::init wraps the physical reader with TracingFileReader when scan IO stats are - // enabled. Prefetch should target the physical cached reader below that tracing wrapper, - // otherwise v2 scans with profiling would silently lose prefetch. - if (auto tracing_reader = std::dynamic_pointer_cast(reader)) { - reader = tracing_reader->inner_reader(); - } - return std::dynamic_pointer_cast(reader); - } - - io::FileReaderSPtr _file_reader; - io::FileReaderSPtr _base_file_reader; - io::FileReaderStats* _file_reader_stats = nullptr; - io::IOContext* _io_ctx = nullptr; - int64_t _pos = 0; - bool _closed = false; - bool _enable_page_cache = false; - bool _merge_range_active = false; - std::string _page_cache_file_key; - mutable std::mutex _page_cache_mutex; - std::vector _page_cache_ranges; - std::shared_ptr _cached_page_range_index; - ParquetPageCacheStats _page_cache_stats; -}; - } // namespace -Status arrow_status_to_doris_status(const arrow::Status& status) { - if (status.ok()) { - return Status::OK(); - } - if (status.IsIOError()) { - return Status::IOError(status.ToString()); - } - if (status.IsInvalid()) { - return Status::InvalidArgument(status.ToString()); - } - return Status::InternalError(status.ToString()); -} - Status ParquetFileContext::open(io::FileReaderSPtr input_file_reader, io::IOContext* io_ctx, bool enable_page_cache, const io::FileDescription& file_description, bool enable_mapping_timestamp_tz, bool enable_mapping_varbinary) { @@ -1016,16 +541,6 @@ void ParquetFileContext::prefetch_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size) { - (void)ranges; - (void)avg_io_size; - (void)profile; - (void)merge_read_slice_size; - return false; -} - bool ParquetFileContext::set_native_random_access_ranges( const std::vector& ranges, size_t avg_io_size, RuntimeProfile* profile, int64_t merge_read_slice_size) { diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index add2b8e18f6fa2..fdaaeaa3c2a5fa 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -15,14 +15,11 @@ #pragma once -#include #include -#include #include #include #include -#include #include #include #include @@ -72,17 +69,6 @@ struct ParquetPageCacheRange { int64_t end_offset() const { return offset + size; } }; -struct ParquetPageCacheReadPlanEntry { - // The exact cached StoragePageCache entry. The final cache key is still exact-range based: - // file key + cached_range.end_offset() + cached_range.offset. - ParquetPageCacheRange cached_range; - // Byte offset inside cached_range to start copying from. - int64_t copy_offset_in_cache = 0; - // Byte offset inside the current ReadAt output buffer to start writing to. - int64_t output_offset = 0; - int64_t copy_size = 0; -}; - struct ParquetPageCacheStats { int64_t read_count = 0; int64_t write_count = 0; @@ -94,39 +80,6 @@ struct ParquetPageCacheStats { namespace detail { -class ParquetPageCacheRangeIndex { -public: - static constexpr size_t DEFAULT_MAX_RANGES = 65536; - - explicit ParquetPageCacheRangeIndex(size_t max_ranges = DEFAULT_MAX_RANGES); - - void insert(ParquetPageCacheRange range); - void erase(ParquetPageCacheRange range); - - std::vector ranges() const; - size_t size() const; - -private: - const size_t _max_ranges; - mutable std::mutex _mutex; - std::vector _ranges; -}; - -class ParquetPageCacheRangeDirectory { -public: - static constexpr size_t DEFAULT_MAX_FILES = 4096; - - explicit ParquetPageCacheRangeDirectory(size_t max_files = DEFAULT_MAX_FILES); - - std::shared_ptr get_or_create(const std::string& file_key); - size_t size() const; - -private: - const size_t _max_files; - mutable std::mutex _mutex; - std::unordered_map> _indexes; -}; - inline constexpr int64_t MAX_SERIALIZED_PARQUET_INDEX_BYTES = 64LL << 20; Status validate_native_footer_size(uint32_t serialized_size, size_t file_size, @@ -141,22 +94,6 @@ bool is_serialized_index_range_safe(size_t file_size, int64_t offset, int64_t le bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); -// Build the copy plan for a ReadAt(position, nbytes) request from the range metadata of -// previously cached entries. -// StoragePageCache cannot do range lookup by itself; it can only lookup an exact key. The -// caller therefore keeps lightweight cached range metadata and uses this function to decide -// which exact cache entries to fetch and which byte spans to copy. -// Examples: -// 1. Subset hit: -// request [120, 150), cached [100, 200) -> copy 30 bytes from cached offset 20. -// 2. Superset hit covered by multiple cached entries: -// request [100, 260), cached [100, 180) and [180, 260) -// -> two copies: [100, 180) to output offset 0, [180, 260) to output offset 80. -// 3. Partial overlap is a miss: -// request [100, 260), cached [100, 180) only -> empty plan, caller reads from file. -std::vector plan_page_cache_range_read( - int64_t position, int64_t nbytes, const std::vector& cached_ranges); - // Keep only byte ranges that are safe to hand to FileReader implementations. Parquet metadata is // expected to contain non-negative offsets and positive compressed sizes, but tests and corrupted // footers can still feed invalid values. Example: [100, 64) is kept, while [-1, 64), [100, 0) and @@ -219,10 +156,6 @@ struct ParquetFileContext { // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Deprecated adapter hook. Native readers use set_native_random_access_ranges(). - bool set_random_access_ranges(const std::vector& ranges, - size_t avg_io_size, RuntimeProfile* profile, - int64_t merge_read_slice_size); // Install the v1-compatible MergeRangeFileReader on the native data-page path. Dictionary // probes must run before this method because their Arrow ReadAt order is independent of the // sequential projected chunk ranges consumed by MergeRangeFileReader. @@ -238,6 +171,4 @@ struct ParquetFileContext { Status close(); }; -Status arrow_status_to_doris_status(const arrow::Status& status); - } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index c0c2630e1532f2..ed542f5f446e14 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -57,68 +57,6 @@ struct ParquetReaderScanState { bool enable_strict_mode = false; }; -int64_t column_chunk_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); -} - -Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { - if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (!column_schema.type_descriptor.unsupported_reason.empty()) { - return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, - column_schema.type_descriptor.unsupported_reason); - } - return Status::OK(); - } - for (const auto& child : column_schema.children) { - DORIS_CHECK(child != nullptr); - RETURN_IF_ERROR(validate_all_projected_leaves_supported(*child)); - } - return Status::OK(); -} - -Status validate_projected_leaves_supported(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex& projection) { - if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE || - projection.project_all_children || projection.children.empty()) { - return validate_all_projected_leaves_supported(column_schema); - } - for (const auto& child_projection : projection.children) { - const auto child_it = - std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { - return child_schema->local_id == child_projection.local_id(); - }); - DORIS_CHECK(child_it != column_schema.children.end()); - RETURN_IF_ERROR(validate_projected_leaves_supported(**child_it, child_projection)); - } - return Status::OK(); -} - -Status validate_requested_columns_supported( - const std::vector>& file_schema, - const format::FileScanRequest& request) { - auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { - const auto local_id = projection.local_id(); - if (local_id == format::ROW_POSITION_COLUMN_ID || - local_id == format::GLOBAL_ROWID_COLUMN_ID) { - return Status::OK(); - } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - DORIS_CHECK(file_schema[local_id] != nullptr); - return validate_projected_leaves_supported(*file_schema[local_id], projection); - }; - for (const auto& column : request.predicate_columns) { - RETURN_IF_ERROR(validate_scan_column(column)); - } - for (const auto& column : request.non_predicate_columns) { - if (!request.is_count_star_placeholder(column.column_id())) { - RETURN_IF_ERROR(validate_scan_column(column)); - } - } - return Status::OK(); -} - const ParquetColumnSchema& projected_root_schema( const std::vector>& file_schema, const format::LocalColumnIndex& projection) { @@ -180,7 +118,7 @@ int timestamp_tz_scale(const ParquetTypeDescriptor& type_descriptor) { bool should_map_to_timestamp_tz(const ParquetColumnSchema& column_schema) { const auto& type_descriptor = column_schema.type_descriptor; - return type_descriptor.physical_type == ::parquet::Type::INT96 || + return type_descriptor.physical_type == tparquet::Type::INT96 || (type_descriptor.is_timestamp && type_descriptor.timestamp_is_adjusted_to_utc); } @@ -262,8 +200,8 @@ static Status find_projected_minmax_leaf(const ParquetColumnSchema& column_schem static Status validate_minmax_aggregate_statistics(const ParquetColumnSchema& column_schema) { switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: // Arrow 17 does not expose Parquet's min/max exactness flags. Binary statistics may be // truncated bounds rather than values present in the file, so they are safe for pruning // but cannot be returned as exact aggregate results. diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index b16af939ae2234..5599b93aa9a623 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -15,8 +15,6 @@ #include "format_v2/parquet/parquet_scan.h" -#include - #include #include #include @@ -101,12 +99,6 @@ namespace { detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request); -int64_t column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { - return column_metadata.has_dictionary_page() - ? cast_set(column_metadata.dictionary_page_offset()) - : cast_set(column_metadata.data_page_offset()); -} - bool is_dictionary_data_encoding(tparquet::Encoding::type encoding) { return encoding == tparquet::Encoding::PLAIN_DICTIONARY || encoding == tparquet::Encoding::RLE_DICTIONARY; @@ -209,35 +201,6 @@ void collect_projected_leaf_column_ids(const ParquetColumnSchema& column_schema, } } -bool is_row_group_outside_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, int row_group_idx) { - if (scan_range.size < 0) { - return false; - } - const int64_t range_start_offset = scan_range.start_offset; - const int64_t range_end_offset = range_start_offset + scan_range.size; - DORIS_CHECK(range_start_offset >= 0); - DORIS_CHECK(range_end_offset >= range_start_offset); - if (range_start_offset == 0 && - (scan_range.file_size < 0 || range_end_offset >= scan_range.file_size)) { - return false; - } - - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - DORIS_CHECK(row_group_metadata->num_columns() > 0); - const auto first_column = row_group_metadata->ColumnChunk(0); - const auto last_column = row_group_metadata->ColumnChunk(row_group_metadata->num_columns() - 1); - DORIS_CHECK(first_column != nullptr); - DORIS_CHECK(last_column != nullptr); - const int64_t row_group_start_offset = column_start_offset(*first_column); - const int64_t row_group_end_offset = - column_start_offset(*last_column) + last_column->total_compressed_size(); - const int64_t row_group_mid_offset = - row_group_start_offset + (row_group_end_offset - row_group_start_offset) / 2; - return row_group_mid_offset < range_start_offset || row_group_mid_offset >= range_end_offset; -} - std::vector request_scan_columns(const format::FileScanRequest& request) { std::vector scan_columns; scan_columns.reserve(request.predicate_columns.size() + request.non_predicate_columns.size()); @@ -344,153 +307,6 @@ Status build_native_prefetch_ranges( } // namespace detail -namespace { - -Status select_row_groups_by_scan_range(const ::parquet::FileMetaData& metadata, - const ParquetScanRange& scan_range, - std::vector* row_group_first_rows, - std::vector* selected_row_groups) { - DORIS_CHECK(row_group_first_rows != nullptr); - DORIS_CHECK(selected_row_groups != nullptr); - row_group_first_rows->assign(metadata.num_row_groups(), 0); - selected_row_groups->clear(); - selected_row_groups->reserve(metadata.num_row_groups()); - int64_t next_row_group_first_row = 0; - for (int row_group_idx = 0; row_group_idx < metadata.num_row_groups(); ++row_group_idx) { - (*row_group_first_rows)[row_group_idx] = next_row_group_first_row; - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows < 0) { - return Status::Corruption("Invalid negative row count in parquet row group {}", - row_group_idx); - } - next_row_group_first_row += row_group_rows; - if (!is_row_group_outside_range(metadata, scan_range, row_group_idx)) { - selected_row_groups->push_back(row_group_idx); - } - } - return Status::OK(); -} - -Status build_row_group_read_plans( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector& selected_row_groups, - const std::vector& row_group_first_rows, RowGroupScanPlan* plan, - const cctz::time_zone* timezone, const RuntimeState* runtime_state, - ParquetFileContext* file_context) { - DORIS_CHECK(plan != nullptr); - plan->row_groups.reserve(selected_row_groups.size()); - std::unordered_set requested_leaf_ids; - if (file_context != nullptr) { - for (const auto& projection : request_scan_columns(request)) { - const auto local_id = projection.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size())) { - continue; - } - collect_projected_leaf_column_ids(*file_schema[local_id], projection, - &requested_leaf_ids); - } - } - for (const auto row_group_idx : selected_row_groups) { - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(static_cast(row_group_idx) < row_group_first_rows.size()); - auto row_group_metadata = metadata.RowGroup(row_group_idx); - DORIS_CHECK(row_group_metadata != nullptr); - const int64_t row_group_rows = row_group_metadata->num_rows(); - if (row_group_rows == 0) { - continue; - } - - RowGroupReadPlan row_group_plan; - row_group_plan.row_group_id = row_group_idx; - row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; - row_group_plan.row_group_rows = row_group_rows; - if (file_context != nullptr) { - std::unordered_map page_indexes; - if (can_use_parquet_page_index(request, runtime_state)) { - RETURN_IF_ERROR(file_context->load_native_page_indexes( - row_group_idx, requested_leaf_ids, &page_indexes, - &plan->pruning_stats.read_page_index_time, - &plan->pruning_stats.parse_page_index_time)); - } - RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( - page_indexes, file_schema, request, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - for (auto& [leaf_column_id, indexes] : page_indexes) { - row_group_plan.offset_indexes.emplace(leaf_column_id, - std::move(indexes.offset_index)); - } - } else { - RETURN_IF_ERROR(select_row_group_ranges_by_page_index( - file_reader, file_schema, request, row_group_idx, row_group_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - } - if (row_group_plan.selected_ranges.empty()) { - continue; - } - plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); - plan->row_groups.push_back(std::move(row_group_plan)); - } - return Status::OK(); -} - -} // namespace - -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const ParquetScanRange& scan_range, bool enable_bloom_filter, - RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state, - ParquetFileContext* file_context) { - DORIS_CHECK(plan != nullptr); - plan->row_groups.clear(); - plan->pruning_stats = ParquetPruningStats {}; - - // Row-group planning flow: - // - // parquet footer row groups - // | - // v - // split byte-range candidates - // | - // v - // row-group metadata pruning - // statistics/ZoneMap -> dictionary -> bloom filter - // | - // v - // page-index pruning per selected row group - // | - // v - // RowGroupReadPlan with selected row ranges - // - // Metadata pruning removes whole row groups before readers are opened. Page index pruning runs - // only for remaining row groups and produces selected row ranges; the scan scheduler later skips - // gaps between those ranges, while row-level VExpr conjuncts still run on loaded batches for - // correctness. - std::vector row_group_first_rows; - std::vector scan_range_selected_row_groups; - RETURN_IF_ERROR(select_row_groups_by_scan_range(metadata, scan_range, &row_group_first_rows, - &scan_range_selected_row_groups)); - - std::vector metadata_selected_row_groups; - RETURN_IF_ERROR(select_row_groups_by_metadata( - metadata, file_reader, file_schema, request, &scan_range_selected_row_groups, - &metadata_selected_row_groups, enable_bloom_filter, &plan->pruning_stats, timezone, - runtime_state)); - - RETURN_IF_ERROR(build_row_group_read_plans(metadata, file_reader, file_schema, request, - metadata_selected_row_groups, row_group_first_rows, - plan, timezone, runtime_state, file_context)); - plan->pruning_stats.selected_row_groups = plan->row_groups.size(); - return Status::OK(); -} - namespace detail { Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& metadata, diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 253f6d1696960c..e1dfcd5b2258d5 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -36,13 +36,6 @@ #include "runtime/runtime_profile.h" #include "storage/segment/condition_cache.h" -namespace parquet { -class FileMetaData; -class ParquetFileReader; -class RowGroupMetaData; -class RowGroupReader; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -124,15 +117,6 @@ struct RowGroupScanPlan { // ============================================================================ // ============================================================================ -Status plan_parquet_row_groups(const ::parquet::FileMetaData& metadata, - ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const ParquetScanRange& scan_range, bool enable_bloom_filter, - RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr, - ParquetFileContext* file_context = nullptr); - Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const std::vector>& file_schema, const format::FileScanRequest& request, diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index c0bb4ef10b4b67..04110306eb6a8f 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -15,15 +15,6 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include @@ -91,8 +82,8 @@ Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, const tparquet::Statistics& statistics) { - const bool binary = type_descriptor.physical_type == ::parquet::Type::BYTE_ARRAY || - type_descriptor.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + const bool binary = type_descriptor.physical_type == tparquet::Type::BYTE_ARRAY || + type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; if (!binary) { return true; } @@ -284,30 +275,6 @@ bool decoded_min_max_is_ordered(const ParquetColumnStatistics& column_statistics return !(column_statistics.max_value < column_statistics.min_value); } -template -bool set_decoded_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics>(statistics); - const auto& min_value = typed_statistics->min(); - const auto& max_value = typed_statistics->max(); - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; - } - } - if (!valid_min_max(min_value, max_value) || - !set_decoded_field(column_schema, value_kind, min_value, &column_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &column_statistics->max_value, - timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); -} - bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedValueKind value_kind, const StringRef& value, Field* field, const cctz::time_zone* timezone) { @@ -318,54 +285,6 @@ bool set_decoded_binary_field(const ParquetColumnSchema& column_schema, DecodedV return read_decoded_field(column_schema, view, field, timezone).ok(); } -bool set_string_min_max(const std::shared_ptr<::parquet::Statistics>& statistics, - const ParquetColumnSchema& column_schema, - ParquetColumnStatistics* column_statistics, - const cctz::time_zone* timezone) { - switch (statistics->physical_type()) { - case ::parquet::Type::BYTE_ARRAY: { - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::ByteArrayType>>( - statistics); - const auto min = ::parquet::ByteArrayToString(typed_statistics->min()); - const auto max = ::parquet::ByteArrayToString(typed_statistics->max()); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - if (column_schema.descriptor == nullptr || column_schema.descriptor->type_length() <= 0) { - return false; - } - auto typed_statistics = - std::static_pointer_cast<::parquet::TypedStatistics<::parquet::FLBAType>>( - statistics); - const int type_length = column_schema.descriptor->type_length(); - const std::string min(reinterpret_cast(typed_statistics->min().ptr), - type_length); - const std::string max(reinterpret_cast(typed_statistics->max().ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &column_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &column_statistics->max_value, timezone)) { - return false; - } - return decoded_min_max_is_ordered(*column_statistics); - } - default: - return false; - } -} - template T load_predicate_value(const char* data) { T value; @@ -418,123 +337,6 @@ std::optional convert_logical_integer_to_physical_int32( return physical_value; } -class ArrowParquetBloomFilterAdapter final : public segment_v2::BloomFilter { -public: - ArrowParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, - const ::parquet::BloomFilter& bloom_filter) - : _column_schema(column_schema), _bloom_filter(bloom_filter) {} - - void add_bytes(const char* buf, size_t size) override { DORIS_CHECK(false); } - - bool test_bytes(const char* buf, size_t size) const override { - if (buf == nullptr) { - return true; - } - // Parquet bloom filters are populated from the physical column carrier, while VExpr - // literals are materialized as Doris logical values. Keep the logical type in - // BloomFilterEvalContext for expression compatibility, and normalize to the Parquet - // physical representation only at this adapter boundary. - switch (_column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - return test_boolean(buf, size); - case ::parquet::Type::INT32: - return test_physical_int32(buf, size); - case ::parquet::Type::INT64: - return test_int64(buf, size); - case ::parquet::Type::FLOAT: - return test_float(buf, size); - case ::parquet::Type::DOUBLE: - return test_double(buf, size); - case ::parquet::Type::BYTE_ARRAY: - return test_byte_array(buf, size); - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return test_fixed_len_byte_array(buf, size); - default: - return true; - } - } - - void set_has_null(bool has_null) override { DORIS_CHECK(!has_null); } - bool has_null() const override { return false; } - void add_hash(uint64_t hash) override { DORIS_CHECK(false); } - bool test_hash(uint64_t hash) const override { return _bloom_filter.FindHash(hash); } - -private: - bool test_boolean(const char* buf, size_t size) const { - if (size == sizeof(bool)) { - const int32_t value = load_predicate_value(buf) ? 1 : 0; - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - if (size == sizeof(int32_t)) { - const int32_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value != 0 ? 1 : 0)); - } - return true; - } - - bool test_physical_int32(const char* buf, size_t size) const { - const auto logical_value = load_predicate_integral_value(buf, size); - if (!logical_value.has_value()) { - return true; - } - const auto physical_value = convert_logical_integer_to_physical_int32( - _column_schema.type_descriptor, *logical_value); - if (!physical_value.has_value()) { - return false; - } - return find_int32(*physical_value); - } - - bool test_int64(const char* buf, size_t size) const { - if (size != sizeof(int64_t)) { - return true; - } - const int64_t value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_float(const char* buf, size_t size) const { - if (size != sizeof(float)) { - return true; - } - const float value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_double(const char* buf, size_t size) const { - if (size != sizeof(double)) { - return true; - } - const double value = load_predicate_value(buf); - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - bool test_byte_array(const char* buf, size_t size) const { - ::parquet::ByteArray value(static_cast(size), - reinterpret_cast(buf)); - return _bloom_filter.FindHash(_bloom_filter.Hash(&value)); - } - - bool test_fixed_len_byte_array(const char* buf, size_t size) const { - if (_column_schema.type_descriptor.fixed_length <= 0) { - return true; - } - if (size != static_cast(_column_schema.type_descriptor.fixed_length)) { - return false; - } - ::parquet::FLBA value(reinterpret_cast(buf)); - return _bloom_filter.FindHash( - _bloom_filter.Hash(&value, _column_schema.type_descriptor.fixed_length)); - } - - bool find_int32(int32_t value) const { - return _bloom_filter.FindHash(_bloom_filter.Hash(value)); - } - - const ParquetColumnSchema& _column_schema; - const ::parquet::BloomFilter& _bloom_filter; -}; - class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { public: NativeParquetBloomFilterAdapter(const ParquetColumnSchema& column_schema, @@ -545,7 +347,7 @@ class NativeParquetBloomFilterAdapter final : public segment_v2::BloomFilter { bool test_bytes(const char* buf, size_t size) const override { if (buf == nullptr || - _column_schema.type_descriptor.physical_type != ::parquet::Type::INT32) { + _column_schema.type_descriptor.physical_type != tparquet::Type::INT32) { return _bloom_filter.test_bytes(buf, size); } const auto logical_value = load_predicate_integral_value(buf, size); @@ -578,14 +380,14 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { return false; } switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - case ::parquet::Type::INT32: - case ::parquet::Type::INT64: - case ::parquet::Type::FLOAT: - case ::parquet::Type::DOUBLE: - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BOOLEAN: + case tparquet::Type::INT32: + case tparquet::Type::INT64: + case tparquet::Type::FLOAT: + case tparquet::Type::DOUBLE: + case tparquet::Type::BYTE_ARRAY: return true; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return column_schema.type_descriptor.is_string_like && column_schema.type_descriptor.fixed_length > 0; default: @@ -593,220 +395,6 @@ bool bloom_filter_supported(const ParquetColumnSchema& column_schema) { } } -bool bloom_filter_excludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - if (!bloom_filter_supported(column_schema)) { - return false; - } - ArrowParquetBloomFilterAdapter adapter(column_schema, bloom_filter); - BloomFilterEvalContext ctx; - ctx.slots.emplace(slot_index, BloomFilterEvalContext::SlotBloomFilter { - .data_type = column_schema.type, - .bloom_filter = &adapter, - }); - return VExprContext::evaluate_bloom_filter(conjuncts, ctx) == ZoneMapFilterResult::kNoMatch; -} - -struct RowGroupBloomFilterCache { - using CacheKey = std::pair; - - ::parquet::BloomFilterReader* bloom_filter_reader = nullptr; - std::map> column_bloom_filters; - std::set loaded_columns; - - ::parquet::BloomFilter* get(int row_group_idx, int leaf_column_id, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_reader == nullptr || leaf_column_id < 0) { - return nullptr; - } - const CacheKey cache_key {row_group_idx, leaf_column_id}; - if (loaded_columns.find(cache_key) == loaded_columns.end()) { - loaded_columns.insert(cache_key); - try { - std::shared_ptr<::parquet::RowGroupBloomFilterReader> row_group_reader; - if (pruning_stats != nullptr) { - SCOPED_RAW_TIMER(&pruning_stats->bloom_filter_read_time); - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } else { - row_group_reader = bloom_filter_reader->RowGroup(row_group_idx); - if (row_group_reader != nullptr) { - column_bloom_filters[cache_key] = - row_group_reader->GetColumnBloomFilter(leaf_column_id); - } - } - } catch (const ::parquet::ParquetException&) { - return nullptr; - } catch (const std::exception&) { - return nullptr; - } - } - auto it = column_bloom_filters.find(cache_key); - return it == column_bloom_filters.end() ? nullptr : it->second.get(); - } -}; - -bool is_dictionary_data_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::PLAIN_DICTIONARY || - encoding == ::parquet::Encoding::RLE_DICTIONARY; -} - -bool is_level_encoding(::parquet::Encoding::type encoding) { - return encoding == ::parquet::Encoding::RLE || encoding == ::parquet::Encoding::BIT_PACKED; -} - -bool is_data_page_type(::parquet::PageType::type page_type) { - return page_type == ::parquet::PageType::DATA_PAGE || - page_type == ::parquet::PageType::DATA_PAGE_V2; -} - -bool is_dictionary_encoded_chunk(const ::parquet::ColumnChunkMetaData& column_metadata) { - if (!column_metadata.has_dictionary_page()) { - return false; - } - - const auto& encoding_stats = column_metadata.encoding_stats(); - if (!encoding_stats.empty()) { - bool has_dictionary_data_page = false; - for (const auto& encoding_stat : encoding_stats) { - if (!is_data_page_type(encoding_stat.page_type) || encoding_stat.count <= 0) { - continue; - } - if (!is_dictionary_data_encoding(encoding_stat.encoding)) { - return false; - } - has_dictionary_data_page = true; - } - return has_dictionary_data_page; - } - - bool has_dictionary_encoding = false; - for (const auto encoding : column_metadata.encodings()) { - if (is_dictionary_data_encoding(encoding)) { - has_dictionary_encoding = true; - continue; - } - if (!is_level_encoding(encoding)) { - return false; - } - } - return has_dictionary_encoding; -} - -bool supports_dictionary_pruning(const ParquetColumnSchema& column_schema, - const ::parquet::ColumnChunkMetaData& column_metadata) { - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.type == nullptr) { - return false; - } - if (!column_schema.type_descriptor.is_string_like) { - return false; - } - if (column_metadata.type() != ::parquet::Type::BYTE_ARRAY && - column_metadata.type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - return false; - } - return true; -} - -} // namespace - -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words) { - DORIS_CHECK(dict_words != nullptr); - dict_words->clear(); - if (file_reader == nullptr || leaf_column_id < 0) { - return false; - } - - auto row_group_reader = file_reader->RowGroup(row_group_idx); - if (row_group_reader == nullptr) { - return false; - } - auto page_reader = row_group_reader->GetColumnPageReader(leaf_column_id); - if (page_reader == nullptr) { - return false; - } - - std::shared_ptr<::parquet::Page> page; - try { - page = page_reader->NextPage(); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; - } - if (page == nullptr || page->type() != ::parquet::PageType::DICTIONARY_PAGE) { - return false; - } - const auto* dictionary_page = static_cast(page.get()); - if (dictionary_page->encoding() != ::parquet::Encoding::PLAIN && - dictionary_page->encoding() != ::parquet::Encoding::PLAIN_DICTIONARY) { - return false; - } - const int32_t dictionary_length = dictionary_page->num_values(); - if (dictionary_length <= 0) { - return false; - } - const auto* dictionary_data = dictionary_page->data(); - const int dictionary_size = dictionary_page->size(); - - dict_words->values.reserve(static_cast(dictionary_length)); - if (column_schema.descriptor->physical_type() == ::parquet::Type::BYTE_ARRAY) { - auto decoder = ::parquet::MakeTypedDecoder<::parquet::ByteArrayType>( - ::parquet::Encoding::PLAIN, column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::ByteArray> byte_array_values(static_cast(dictionary_length)); - if (decoder->Decode(byte_array_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(byte_array_values[dict_idx].ptr), - byte_array_values[dict_idx].len); - } - dict_words->build_refs(); - return true; - } - if (column_schema.descriptor->physical_type() == ::parquet::Type::FIXED_LEN_BYTE_ARRAY) { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; - } - auto decoder = ::parquet::MakeTypedDecoder<::parquet::FLBAType>(::parquet::Encoding::PLAIN, - column_schema.descriptor); - decoder->SetData(dictionary_length, dictionary_data, dictionary_size); - std::vector<::parquet::FixedLenByteArray> flba_values( - static_cast(dictionary_length)); - if (decoder->Decode(flba_values.data(), dictionary_length) != dictionary_length) { - return false; - } - for (int32_t dict_idx = 0; dict_idx < dictionary_length; ++dict_idx) { - dict_words->values.emplace_back( - reinterpret_cast(flba_values[dict_idx].ptr), type_length); - } - dict_words->build_refs(); - return true; - } - return false; -} - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words) { - std::vector fields; - fields.reserve(dict_words.refs.size()); - for (const auto& ref : dict_words.refs) { - fields.push_back(Field::create_field(String(ref.data, ref.size))); - } - return fields; -} - -namespace { - const ParquetColumnSchema* resolve_local_leaf_schema( const std::vector>& schema, const format::LocalColumnId file_column_id) { @@ -922,54 +510,6 @@ std::shared_ptr ParquetStatisticsUtils::MakeZoneMap( return make_zonemap_from_statistics(statistics); } -ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, const cctz::time_zone* timezone) { - ParquetColumnStatistics result; - if (statistics == nullptr) { - return result; - } - - result.has_null = !statistics->HasNullCount() || statistics->null_count() > 0; - result.has_not_null = statistics->num_values() > 0 || statistics->HasMinMax(); - result.has_null_count = statistics->HasNullCount(); - if (!result.has_not_null || !statistics->HasMinMax()) { - return result; - } - - DORIS_CHECK(column_schema.type != nullptr); - switch (statistics->physical_type()) { - case ::parquet::Type::BOOLEAN: - result.has_min_max = set_decoded_min_max<::parquet::BooleanType>( - statistics, column_schema, DecodedValueKind::BOOL, &result, timezone); - return result; - case ::parquet::Type::INT32: - result.has_min_max = set_decoded_min_max<::parquet::Int32Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::INT64: - result.has_min_max = set_decoded_min_max<::parquet::Int64Type>( - statistics, column_schema, decoded_value_kind(column_schema.type_descriptor), - &result, timezone); - return result; - case ::parquet::Type::FLOAT: - result.has_min_max = set_decoded_min_max<::parquet::FloatType>( - statistics, column_schema, DecodedValueKind::FLOAT, &result, timezone); - return result; - case ::parquet::Type::DOUBLE: - result.has_min_max = set_decoded_min_max<::parquet::DoubleType>( - statistics, column_schema, DecodedValueKind::DOUBLE, &result, timezone); - return result; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result.has_min_max = set_string_min_max(statistics, column_schema, &result, timezone); - return result; - default: - return result; - } -} - ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, int64_t column_value_count, const cctz::time_zone* timezone) { @@ -1013,12 +553,6 @@ ParquetColumnStatistics ParquetStatisticsUtils::TransformColumnStatistics( return result; } -bool ParquetStatisticsUtils::BloomFilterExcludes(const ParquetColumnSchema& column_schema, - int slot_index, const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter) { - return bloom_filter_excludes(column_schema, slot_index, conjuncts, bloom_filter); -} - bool ParquetStatisticsUtils::NativeBloomFilterExcludes( const ParquetColumnSchema& column_schema, int slot_index, const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter) { @@ -1036,299 +570,50 @@ bool ParquetStatisticsUtils::NativeBloomFilterExcludes( namespace { -ParquetRowGroupPruneReason dictionary_prune_reason( - const ::parquet::RowGroupMetaData& row_group, ::parquet::ParquetFileReader* file_reader, - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request) { - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_dictionary_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { - continue; - } - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk == nullptr || - !supports_dictionary_pruning(*column_schema, *column_chunk) || - !is_dictionary_encoded_chunk(*column_chunk)) { - continue; +void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex* projection, + std::set* leaf_column_ids) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (column_schema.leaf_column_id >= 0) { + leaf_column_ids->insert(column_schema.leaf_column_id); } - - ParquetDictionaryWords dict_words; - if (!read_dictionary_words(file_reader, row_group_idx, column_schema->leaf_column_id, - *column_schema, &dict_words)) { + return; + } + for (const auto& child_schema : column_schema.children) { + if (!format::is_child_projected(projection, child_schema->local_id)) { continue; } - DictionaryEvalContext ctx; - ctx.slots.emplace(slot_index, DictionaryEvalContext::SlotDictionary { - .data_type = column_schema->type, - .values = dictionary_fields_from_words(dict_words), - }); - if (VExprContext::evaluate_dictionary_filter(conjuncts, ctx) == - ZoneMapFilterResult::kNoMatch) { - return ParquetRowGroupPruneReason::DICTIONARY; - } + collect_filtered_leaf_ids(*child_schema, + format::find_child_projection(projection, child_schema->local_id), + leaf_column_ids); } - return ParquetRowGroupPruneReason::NONE; } -ParquetRowGroupPruneReason bloom_filter_prune_reason( - int row_group_idx, const std::vector>& file_schema, - const format::FileScanRequest& request, RowGroupBloomFilterCache* bloom_filter_cache, - ParquetPruningStats* pruning_stats) { - if (bloom_filter_cache == nullptr) { - return ParquetRowGroupPruneReason::NONE; +bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { + DORIS_CHECK(column_schema.type != nullptr); + // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in + // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. + return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; +} + +bool check_native_statistics(const tparquet::RowGroup& row_group, + const std::vector>& file_schema, + const format::FileScanRequest& request, + ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { + const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); + if (slot_indexes.empty()) { + return false; } - const auto conjuncts_by_slot = collect_conjuncts_by_single_slot( - request.conjuncts, expr_zonemap::single_slot_bloom_filter_index); - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { + ZoneMapEvalContext ctx; + for (const int slot_index : slot_indexes) { const auto file_column_id = file_column_id_by_block_position(request, slot_index); if (!file_column_id.has_value()) { continue; } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); if (column_schema == nullptr || column_schema->type == nullptr || - !bloom_filter_supported(*column_schema)) { - continue; - } - auto* bloom_filter = bloom_filter_cache->get(row_group_idx, column_schema->leaf_column_id, - pruning_stats); - if (bloom_filter == nullptr) { - continue; - } - if (ParquetStatisticsUtils::BloomFilterExcludes(*column_schema, slot_index, conjuncts, - *bloom_filter)) { - return ParquetRowGroupPruneReason::BLOOM_FILTER; - } - } - return ParquetRowGroupPruneReason::NONE; -} - -void init_bloom_filter_cache(::parquet::ParquetFileReader* file_reader, bool enable_bloom_filter, - RowGroupBloomFilterCache* bloom_filter_cache) { - DORIS_CHECK(bloom_filter_cache != nullptr); - if (!enable_bloom_filter || file_reader == nullptr) { - return; - } - try { - bloom_filter_cache->bloom_filter_reader = &file_reader->GetBloomFilterReader(); - } catch (const ::parquet::ParquetException&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } catch (const std::exception&) { - bloom_filter_cache->bloom_filter_reader = nullptr; - } -} - -bool check_statistics(const ::parquet::RowGroupMetaData& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { - const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); - if (slot_indexes.empty()) { - return false; - } - - ZoneMapEvalContext ctx; - for (const int slot_index : slot_indexes) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr) { - continue; - } - - std::shared_ptr zone_map; - DCHECK_LT(column_schema->leaf_column_id, row_group.num_columns()); - auto column_chunk = row_group.ColumnChunk(column_schema->leaf_column_id); - if (column_chunk != nullptr) { - zone_map = ParquetStatisticsUtils::MakeZoneMap( - ParquetStatisticsUtils::TransformColumnStatistics( - *column_schema, column_chunk->statistics(), timezone)); - } - add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); - } - - const auto result = VExprContext::evaluate_zonemap_filter(request.conjuncts, ctx); - accumulate_zonemap_stats(ctx, pruning_stats); - return result == ZoneMapFilterResult::kNoMatch; -} - -void collect_filtered_leaf_ids(const ParquetColumnSchema& column_schema, - const format::LocalColumnIndex* projection, - std::set* leaf_column_ids) { - if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { - if (column_schema.leaf_column_id >= 0) { - leaf_column_ids->insert(column_schema.leaf_column_id); - } - return; - } - for (const auto& child_schema : column_schema.children) { - if (!format::is_child_projected(projection, child_schema->local_id)) { - continue; - } - collect_filtered_leaf_ids(*child_schema, - format::find_child_projection(projection, child_schema->local_id), - leaf_column_ids); - } -} - -int64_t requested_compressed_bytes( - const ::parquet::RowGroupMetaData& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request) { - std::set leaf_column_ids; - auto collect_projection = [&](const format::LocalColumnIndex& projection) { - const int32_t local_id = projection.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size()) || - file_schema[local_id] == nullptr) { - return; - } - collect_filtered_leaf_ids(*file_schema[local_id], &projection, &leaf_column_ids); - }; - for (const auto& projection : request.predicate_columns) { - collect_projection(projection); - } - for (const auto& projection : request.non_predicate_columns) { - collect_projection(projection); - } - - int64_t bytes = 0; - for (const int leaf_column_id : leaf_column_ids) { - if (leaf_column_id < 0 || leaf_column_id >= row_group.num_columns()) { - continue; - } - const auto column_chunk = row_group.ColumnChunk(leaf_column_id); - if (column_chunk != nullptr && column_chunk->total_compressed_size() > 0) { - bytes += column_chunk->total_compressed_size(); - } - } - return bytes; -} - -Status select_row_groups_by_metadata_impl( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - int64_t row_group_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &row_group_filter_time_sink - : &pruning_stats->row_group_filter_time); - if (selected_row_groups == nullptr) { - return Status::InvalidArgument("selected_row_groups is null"); - } - selected_row_groups->clear(); - - const int num_row_groups = metadata.num_row_groups(); - const auto candidate_size = candidate_row_groups == nullptr - ? static_cast(num_row_groups) - : candidate_row_groups->size(); - if (pruning_stats != nullptr) { - // Scan-range ownership is decided before metadata pruning. Count only row groups owned by - // this split so a file divided into multiple splits does not report the full-file total and - // out-of-split groups once per split. - pruning_stats->total_row_groups = cast_set(candidate_size); - } - selected_row_groups->reserve(candidate_size); - RowGroupBloomFilterCache bloom_filter_cache; - init_bloom_filter_cache(file_reader, enable_bloom_filter, &bloom_filter_cache); - for (size_t candidate_idx = 0; candidate_idx < candidate_size; ++candidate_idx) { - const int row_group_idx = candidate_row_groups == nullptr - ? static_cast(candidate_idx) - : (*candidate_row_groups)[candidate_idx]; - DORIS_CHECK(row_group_idx >= 0); - DORIS_CHECK(row_group_idx < num_row_groups); - auto row_group = metadata.RowGroup(row_group_idx); - if (row_group == nullptr) { - selected_row_groups->push_back(row_group_idx); - continue; - } - ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; - if (has_expr_zonemap_filter(request, runtime_state) && - check_statistics(*row_group, file_schema, request, pruning_stats, timezone)) { - prune_reason = ParquetRowGroupPruneReason::STATISTICS; - } - - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = dictionary_prune_reason(*row_group, file_reader, row_group_idx, - file_schema, request); - if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = bloom_filter_prune_reason(row_group_idx, file_schema, request, - &bloom_filter_cache, pruning_stats); - } - } - - if (prune_reason != ParquetRowGroupPruneReason::NONE) { - if (pruning_stats != nullptr) { - pruning_stats->filtered_group_rows += row_group->num_rows(); - // FilteredBytes must describe the IO actually avoided by this scan projection; - // counting every physical child overstates savings for nested-column projection. - pruning_stats->filtered_bytes += - requested_compressed_bytes(*row_group, file_schema, request); - if (prune_reason == ParquetRowGroupPruneReason::STATISTICS) { - ++pruning_stats->filtered_row_groups_by_statistics; - } else if (prune_reason == ParquetRowGroupPruneReason::DICTIONARY) { - ++pruning_stats->filtered_row_groups_by_dictionary; - } else if (prune_reason == ParquetRowGroupPruneReason::BLOOM_FILTER) { - ++pruning_stats->filtered_row_groups_by_bloom_filter; - } - } - continue; - } - selected_row_groups->push_back(row_group_idx); - } - return Status::OK(); -} - -} // namespace - -Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - return select_row_groups_by_metadata_impl( - metadata, file_reader, file_schema, request, candidate_row_groups, selected_row_groups, - enable_bloom_filter, pruning_stats, timezone, runtime_state); -} - -namespace { - -bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_schema) { - DORIS_CHECK(column_schema.type != nullptr); - // Raw VARBINARY file slots may feed table-side STRING casts. Footer/page metadata is still in - // the pre-cast domain, so using it for a rewritten table predicate can cause false negatives. - return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; -} - -bool check_native_statistics(const tparquet::RowGroup& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { - const auto slot_indexes = collect_expr_zonemap_slot_indexes(request.conjuncts); - if (slot_indexes.empty()) { - return false; - } - ZoneMapEvalContext ctx; - for (const int slot_index : slot_indexes) { - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - continue; - } - const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (column_schema == nullptr || column_schema->type == nullptr || - !native_metadata_predicate_is_type_safe(*column_schema) || - column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { + !native_metadata_predicate_is_type_safe(*column_schema) || + column_schema->leaf_column_id >= static_cast(row_group.columns.size())) { continue; } const auto& chunk = row_group.columns[column_schema->leaf_column_id]; @@ -1615,162 +900,6 @@ Status select_row_groups_by_metadata( namespace { -template -bool set_page_decoded_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - DecodedValueKind value_kind, ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - const auto typed_index = - std::static_pointer_cast<::parquet::TypedColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const typename ParquetDType::c_type min_value = typed_index->min_values()[page_idx]; - const typename ParquetDType::c_type max_value = typed_index->max_values()[page_idx]; - if constexpr (std::is_same_v) { - if (!timestamp_min_max_is_safe(column_schema, min_value, max_value, timezone)) { - return false; - } - } - if (!valid_min_max(min_value, max_value)) { - // A NaN invalidates only this page's bounds, not the ColumnIndex itself. Keep the page - // conservatively by returning usable null-count statistics with has_min_max=false, while - // allowing later pages with finite bounds to remain eligible for pruning. - return true; - } - if (!set_decoded_field(column_schema, value_kind, min_value, &page_statistics->min_value, - timezone) || - !set_decoded_field(column_schema, value_kind, max_value, &page_statistics->max_value, - timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; -} - -bool set_page_string_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BYTE_ARRAY: { - const auto typed_index = - std::static_pointer_cast<::parquet::ByteArrayColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const auto min = ::parquet::ByteArrayToString(typed_index->min_values()[page_idx]); - const auto max = ::parquet::ByteArrayToString(typed_index->max_values()[page_idx]); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; - } - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { - const int type_length = column_schema.descriptor->type_length(); - if (type_length <= 0) { - return false; - } - const auto typed_index = std::static_pointer_cast<::parquet::FLBAColumnIndex>(column_index); - if (page_idx >= typed_index->min_values().size() || - page_idx >= typed_index->max_values().size()) { - return false; - } - const std::string min( - reinterpret_cast(typed_index->min_values()[page_idx].ptr), - type_length); - const std::string max( - reinterpret_cast(typed_index->max_values()[page_idx].ptr), - type_length); - if (!set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(min.data(), min.size()), - &page_statistics->min_value, timezone) || - !set_decoded_binary_field(column_schema, DecodedValueKind::FIXED_BINARY, - StringRef(max.data(), max.size()), - &page_statistics->max_value, timezone)) { - return false; - } - if (!decoded_min_max_is_ordered(*page_statistics)) { - return true; - } - page_statistics->has_min_max = true; - return true; - } - default: - return false; - } -} - -bool set_page_min_max(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - DORIS_CHECK(column_schema.type != nullptr); - switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: - return set_page_decoded_min_max<::parquet::BooleanType>(column_index, column_schema, - page_idx, DecodedValueKind::BOOL, - page_statistics, timezone); - case ::parquet::Type::INT32: - return set_page_decoded_min_max<::parquet::Int32Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::INT64: - return set_page_decoded_min_max<::parquet::Int64Type>( - column_index, column_schema, page_idx, - decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::FLOAT: - return set_page_decoded_min_max<::parquet::FloatType>(column_index, column_schema, page_idx, - DecodedValueKind::FLOAT, - page_statistics, timezone); - case ::parquet::Type::DOUBLE: - return set_page_decoded_min_max<::parquet::DoubleType>(column_index, column_schema, - page_idx, DecodedValueKind::DOUBLE, - page_statistics, timezone); - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - return set_page_string_min_max(column_index, column_schema, page_idx, page_statistics, - timezone); - default: - return false; - } -} - -bool build_page_statistics(const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, - const cctz::time_zone* timezone) { - DORIS_CHECK(page_statistics != nullptr); - *page_statistics = ParquetColumnStatistics {}; - - const auto& null_pages = column_index->null_pages(); - if (!column_index->has_null_counts() || page_idx >= null_pages.size() || - page_idx >= column_index->null_counts().size()) { - return false; - } - - page_statistics->has_null_count = true; - page_statistics->has_null = column_index->null_counts()[page_idx] > 0; - page_statistics->has_not_null = !null_pages[page_idx]; - if (!page_statistics->has_not_null) { - return true; - } - return set_page_min_max(column_index, column_schema, page_idx, page_statistics, timezone); -} - std::vector intersect_ranges(const std::vector& left, const std::vector& right) { std::vector result; @@ -1803,19 +932,6 @@ int64_t count_range_rows(const std::vector& ranges) { return rows; } -RowRange page_row_range(const ::parquet::OffsetIndex& offset_index, size_t page_idx, - int64_t row_group_rows) { - const auto& page_locations = offset_index.page_locations(); - const int64_t start = page_locations[page_idx].first_row_index; - const int64_t end = page_idx + 1 == page_locations.size() - ? row_group_rows - : page_locations[page_idx + 1].first_row_index; - DORIS_CHECK(start >= 0); - DORIS_CHECK(end >= start); - DORIS_CHECK(end <= row_group_rows); - return RowRange {start, end - start}; -} - void append_row_range(const RowRange& range, std::vector* ranges) { if (range.length == 0) { return; @@ -1830,95 +946,6 @@ void append_row_range(const RowRange& range, std::vector* ranges) { ranges->push_back(range); } -std::optional< - std::pair, std::shared_ptr<::parquet::OffsetIndex>>> -load_page_indexes_for_slot(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, - const ParquetColumnSchema** column_schema) { - DORIS_CHECK(column_schema != nullptr); - *column_schema = nullptr; - const auto file_column_id = file_column_id_by_block_position(request, slot_index); - if (!file_column_id.has_value()) { - return std::nullopt; - } - *column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); - if (*column_schema == nullptr || (*column_schema)->descriptor == nullptr) { - return std::nullopt; - } - - try { - auto column_index = row_group->GetColumnIndex((*column_schema)->leaf_column_id); - auto offset_index = row_group->GetOffsetIndex((*column_schema)->leaf_column_id); - if (column_index == nullptr || offset_index == nullptr || - column_index->null_pages().size() != offset_index->page_locations().size()) { - return std::nullopt; - } - return std::make_pair(std::move(column_index), std::move(offset_index)); - } catch (const ::parquet::ParquetException&) { - return std::nullopt; - } catch (const std::exception&) { - return std::nullopt; - } -} - -bool select_ranges_for_expr_zonemap( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, int slot_index, const VExprContextSPtrs& conjuncts, - int64_t row_group_rows, std::vector* ranges, ParquetPruningStats* pruning_stats, - const cctz::time_zone* timezone) { - DORIS_CHECK(ranges != nullptr); - if (conjuncts.empty()) { - return false; - } - const ParquetColumnSchema* column_schema = nullptr; - int64_t parse_page_index_time_sink = 0; - std::optional, - std::shared_ptr<::parquet::OffsetIndex>>> - page_indexes; - { - // Arrow materializes the serialized page-index objects lazily in these getters, so keep - // that cost separate from predicate evaluation when diagnosing a slow page-index scan. - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &parse_page_index_time_sink - : &pruning_stats->parse_page_index_time); - page_indexes = load_page_indexes_for_slot(row_group, file_schema, request, slot_index, - &column_schema); - } - if (!page_indexes.has_value()) { - return false; - } - const auto& [column_index, offset_index] = *page_indexes; - - ranges->clear(); - ZoneMapEvalStats page_stats; - const auto page_count = offset_index->page_locations().size(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - ParquetColumnStatistics page_statistics; - if (!ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *column_schema, page_idx, &page_statistics, timezone)) { - ranges->clear(); - return false; - } - - ZoneMapEvalContext ctx; - add_slot_zonemap(&ctx, slot_index, column_schema->type, - ParquetStatisticsUtils::MakeZoneMap(page_statistics)); - const auto result = VExprContext::evaluate_zonemap_filter(conjuncts, ctx); - page_stats.merge_page_eval_stats(ctx.stats); - if (result == ZoneMapFilterResult::kNoMatch) { - continue; - } - append_row_range(page_row_range(*offset_index, page_idx, row_group_rows), ranges); - } - if (pruning_stats != nullptr) { - pruning_stats->expr_zonemap_unusable_evals += page_stats.unusable_zonemap_eval_count; - pruning_stats->in_zonemap_point_check_count += page_stats.in_zonemap_point_check_count; - pruning_stats->in_zonemap_range_only_count += page_stats.in_zonemap_range_only_count; - } - return true; -} - bool ranges_intersect(const std::vector& ranges, const RowRange& range) { const int64_t range_end = range.start + range.length; for (const auto& selected_range : ranges) { @@ -1978,180 +1005,6 @@ void collect_request_leaf_schemas( } } -bool build_page_skip_plan_for_leaf( - const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const ParquetColumnSchema& column_schema, const std::vector& selected_ranges, - int64_t row_group_rows, ParquetPageSkipPlan* page_skip_plan) { - DORIS_CHECK(page_skip_plan != nullptr); - *page_skip_plan = ParquetPageSkipPlan {}; - if (column_schema.kind != ParquetColumnSchemaKind::PRIMITIVE || - column_schema.descriptor == nullptr || column_schema.leaf_column_id < 0 || - column_schema.descriptor->max_repetition_level() != 0) { - return false; - } - - std::shared_ptr<::parquet::OffsetIndex> offset_index; - try { - offset_index = row_group->GetOffsetIndex(column_schema.leaf_column_id); - } catch (const ::parquet::ParquetException&) { - return false; - } catch (const std::exception&) { - return false; - } - if (offset_index == nullptr) { - return false; - } - - const auto page_count = offset_index->page_locations().size(); - page_skip_plan->leaf_column_id = column_schema.leaf_column_id; - page_skip_plan->skipped_pages.resize(page_count); - page_skip_plan->skipped_page_compressed_sizes.resize(page_count); - const auto& page_locations = offset_index->page_locations(); - for (size_t page_idx = 0; page_idx < page_count; ++page_idx) { - const RowRange row_range = page_row_range(*offset_index, page_idx, row_group_rows); - if (row_range.length == 0 || ranges_intersect(selected_ranges, row_range)) { - continue; - } - page_skip_plan->skipped_pages[page_idx] = 1; - page_skip_plan->skipped_page_compressed_sizes[page_idx] = - page_locations[page_idx].compressed_page_size; - append_row_range(row_range, &page_skip_plan->skipped_ranges); - } - if (page_skip_plan->empty()) { - *page_skip_plan = ParquetPageSkipPlan {}; - return false; - } - return true; -} - -void build_page_skip_plans(const std::shared_ptr<::parquet::RowGroupPageIndexReader>& row_group, - const std::vector>& file_schema, - const format::FileScanRequest& request, - const std::vector& selected_ranges, int64_t row_group_rows, - std::map* page_skip_plans, - ParquetPruningStats* pruning_stats) { - DORIS_CHECK(page_skip_plans != nullptr); - page_skip_plans->clear(); - std::vector leaf_schemas; - collect_request_leaf_schemas(file_schema, request, &leaf_schemas); - for (const auto* leaf_schema : leaf_schemas) { - DORIS_CHECK(leaf_schema != nullptr); - ParquetPageSkipPlan page_skip_plan; - int64_t parse_page_index_time_sink = 0; - bool has_skip_plan = false; - { - // Offset indexes for output-only columns may not have been touched by ZoneMap - // filtering; include their lazy materialization in the same parse timer. - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &parse_page_index_time_sink - : &pruning_stats->parse_page_index_time); - has_skip_plan = build_page_skip_plan_for_leaf(row_group, *leaf_schema, selected_ranges, - row_group_rows, &page_skip_plan); - } - if (has_skip_plan) { - page_skip_plans->emplace(page_skip_plan.leaf_column_id, std::move(page_skip_plan)); - } - } -} - -} // namespace - -bool ParquetStatisticsUtils::TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone) { - return build_page_statistics(column_index, column_schema, page_idx, page_statistics, timezone); -} - -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, - std::vector* selected_ranges, std::map* page_skip_plans, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state) { - int64_t page_index_filter_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &page_index_filter_time_sink - : &pruning_stats->page_index_filter_time); - DORIS_CHECK(selected_ranges != nullptr); - selected_ranges->clear(); - if (page_skip_plans != nullptr) { - page_skip_plans->clear(); - } - if (row_group_rows <= 0) { - return Status::OK(); - } - selected_ranges->push_back(RowRange {0, row_group_rows}); - if (!config::enable_parquet_page_index || !has_expr_zonemap_filter(request, runtime_state) || - file_reader == nullptr) { - return Status::OK(); - } - - std::shared_ptr<::parquet::PageIndexReader> page_index_reader; - std::shared_ptr<::parquet::RowGroupPageIndexReader> row_group_index_reader; - try { - if (pruning_stats != nullptr) { - ++pruning_stats->page_index_read_calls; - } - { - int64_t read_page_index_time_sink = 0; - SCOPED_RAW_TIMER(pruning_stats == nullptr ? &read_page_index_time_sink - : &pruning_stats->read_page_index_time); - page_index_reader = file_reader->GetPageIndexReader(); - if (page_index_reader == nullptr) { - return Status::OK(); - } - row_group_index_reader = page_index_reader->RowGroup(row_group_idx); - } - } catch (const ::parquet::ParquetException&) { - return Status::OK(); - } catch (const std::exception&) { - return Status::OK(); - } - if (row_group_index_reader == nullptr) { - return Status::OK(); - } - - std::map conjuncts_by_slot; - for (const auto& conjunct : request.conjuncts) { - const auto slot_index = expr_zonemap::single_slot_zonemap_index(conjunct); - if (slot_index >= 0) { - conjuncts_by_slot[slot_index].push_back(conjunct); - } - } - - for (const auto& [slot_index, conjuncts] : conjuncts_by_slot) { - std::vector filter_ranges; - if (!select_ranges_for_expr_zonemap(row_group_index_reader, file_schema, request, - slot_index, conjuncts, row_group_rows, &filter_ranges, - pruning_stats, timezone)) { - continue; - } - *selected_ranges = intersect_ranges(*selected_ranges, filter_ranges); - if (selected_ranges->empty()) { - if (page_skip_plans != nullptr) { - page_skip_plans->clear(); - } - if (pruning_stats != nullptr) { - pruning_stats->filtered_page_rows += row_group_rows; - ++pruning_stats->filtered_row_groups_by_page_index; - } - return Status::OK(); - } - } - if (page_skip_plans != nullptr) { - build_page_skip_plans(row_group_index_reader, file_schema, request, *selected_ranges, - row_group_rows, page_skip_plans, pruning_stats); - } - if (pruning_stats != nullptr) { - const int64_t selected_rows = count_range_rows(*selected_ranges); - DORIS_CHECK(selected_rows <= row_group_rows); - pruning_stats->filtered_page_rows += row_group_rows - selected_rows; - } - return Status::OK(); -} - -namespace { - template bool set_native_page_scalar_min_max(const tparquet::ColumnIndex& column_index, const ParquetColumnSchema& column_schema, size_t page_idx, @@ -2205,36 +1058,36 @@ bool build_native_page_statistics(const tparquet::ColumnIndex& column_index, return true; } switch (column_schema.type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: + case tparquet::Type::BOOLEAN: return set_native_page_scalar_min_max(column_index, column_schema, page_idx, DecodedValueKind::BOOL, page_statistics, timezone); - case ::parquet::Type::INT32: + case tparquet::Type::INT32: return set_native_page_scalar_min_max( column_index, column_schema, page_idx, decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::INT64: + case tparquet::Type::INT64: return set_native_page_scalar_min_max( column_index, column_schema, page_idx, decoded_value_kind(column_schema.type_descriptor), page_statistics, timezone); - case ::parquet::Type::FLOAT: + case tparquet::Type::FLOAT: return set_native_page_scalar_min_max(column_index, column_schema, page_idx, DecodedValueKind::FLOAT, page_statistics, timezone); - case ::parquet::Type::DOUBLE: + case tparquet::Type::DOUBLE: return set_native_page_scalar_min_max(column_index, column_schema, page_idx, DecodedValueKind::DOUBLE, page_statistics, timezone); - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: { + case tparquet::Type::BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: { if (page_idx >= column_index.min_values.size() || page_idx >= column_index.max_values.size()) { return false; } const auto& min_value = column_index.min_values[page_idx]; const auto& max_value = column_index.max_values[page_idx]; - const bool fixed = column_schema.type_descriptor.physical_type == - ::parquet::Type::FIXED_LEN_BYTE_ARRAY; + const bool fixed = + column_schema.type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; if (fixed && (column_schema.type_descriptor.fixed_length <= 0 || min_value.size() != static_cast(column_schema.type_descriptor.fixed_length) || diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index ca47b343bcc3f9..6b13444b0aa818 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -32,14 +32,6 @@ #include "format_v2/file_reader.h" #include "format_v2/parquet/selection_vector.h" -namespace parquet { -class BloomFilter; -class ColumnIndex; -class FileMetaData; -class ParquetFileReader; -class Statistics; -} // namespace parquet - namespace cctz { class time_zone; } // namespace cctz @@ -69,36 +61,6 @@ bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, // ============================================================================ // ============================================================================ -struct ParquetDictionaryWords { - std::vector values; - std::vector refs; - - void clear() { - values.clear(); - refs.clear(); - } - - void build_refs() { - refs.clear(); - refs.reserve(values.size()); - for (const auto& value : values) { - refs.emplace_back(value.data(), value.size()); - } - } -}; - -// Reads the PLAIN dictionary page for BYTE_ARRAY/FIXED_LEN_BYTE_ARRAY columns and owns copied -// dictionary bytes in `values`. Both row-group pruning and row-level dictionary predicates use this -// helper so they agree on dictionary id -> Doris string value mapping. -bool read_dictionary_words(::parquet::ParquetFileReader* file_reader, int row_group_idx, - int leaf_column_id, const ParquetColumnSchema& column_schema, - ParquetDictionaryWords* dict_words); - -std::vector dictionary_fields_from_words(const ParquetDictionaryWords& dict_words); - -// ============================================================================ -// ============================================================================ - struct ParquetPruningStats { int64_t total_row_groups = 0; // total row groups in the file int64_t selected_row_groups = 0; // row groups selected after pruning @@ -151,37 +113,15 @@ struct ParquetStatisticsUtils { static std::shared_ptr MakeZoneMap( const ParquetColumnStatistics& statistics); - static ParquetColumnStatistics TransformColumnStatistics( - const ParquetColumnSchema& column_schema, - const std::shared_ptr<::parquet::Statistics>& statistics, - const cctz::time_zone* timezone = nullptr); - static ParquetColumnStatistics TransformColumnStatistics( const ParquetColumnSchema& column_schema, const tparquet::Statistics* statistics, int64_t column_value_count, const cctz::time_zone* timezone = nullptr); - static bool TransformColumnIndexStatistics( - const std::shared_ptr<::parquet::ColumnIndex>& column_index, - const ParquetColumnSchema& column_schema, size_t page_idx, - ParquetColumnStatistics* page_statistics, const cctz::time_zone* timezone = nullptr); - - static bool BloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, - const VExprContextSPtrs& conjuncts, - const ::parquet::BloomFilter& bloom_filter); - static bool NativeBloomFilterExcludes(const ParquetColumnSchema& column_schema, int slot_index, const VExprContextSPtrs& conjuncts, const segment_v2::BloomFilter& bloom_filter); }; -Status select_row_groups_by_metadata( - const ::parquet::FileMetaData& metadata, ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector* candidate_row_groups, - std::vector* selected_row_groups, bool enable_bloom_filter, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); - Status select_row_groups_by_metadata( const tparquet::FileMetaData& metadata, const std::vector>& file_schema, @@ -190,14 +130,6 @@ Status select_row_groups_by_metadata( ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr); -Status select_row_group_ranges_by_page_index( - ::parquet::ParquetFileReader* file_reader, - const std::vector>& file_schema, - const format::FileScanRequest& request, int row_group_idx, int64_t row_group_rows, - std::vector* selected_ranges, std::map* page_skip_plans, - ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr); - Status select_row_group_ranges_by_native_page_index( const std::unordered_map& page_indexes, const std::vector>& file_schema, diff --git a/be/src/format_v2/parquet/parquet_type.cpp b/be/src/format_v2/parquet/parquet_type.cpp index 462908c119dedf..be5ecd1c8b0772 100644 --- a/be/src/format_v2/parquet/parquet_type.cpp +++ b/be/src/format_v2/parquet/parquet_type.cpp @@ -17,315 +17,31 @@ #include "format_v2/parquet/parquet_type.h" -#include - -#include -#include - -#include "core/data_type/data_type_factory.hpp" -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/primitive_type.h" - namespace doris::format::parquet { -namespace { - -DataTypePtr create_type(PrimitiveType type, bool nullable, int precision = 0, int scale = 0) { - return DataTypeFactory::instance().create_data_type(type, nullable, precision, scale); -} - -PrimitiveType decimal_primitive_type(int precision) { - return precision > 38 ? TYPE_DECIMAL256 : TYPE_DECIMAL128I; -} - -void mark_decimal(const ::parquet::ColumnDescriptor* column, int precision, int scale, - ParquetTypeDescriptor* result) { - result->is_decimal = true; - result->decimal_precision = precision; - result->decimal_scale = scale; - switch (column->physical_type()) { - case ::parquet::Type::INT32: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT32; - break; - case ::parquet::Type::INT64: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_INT64; - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - result->extra_type_info = ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY; - break; - default: - result->extra_type_info = ParquetExtraTypeInfo::NONE; - break; - } -} - -void mark_integer(int bit_width, bool is_signed, ParquetTypeDescriptor* result) { - result->integer_bit_width = bit_width; - result->is_unsigned_integer = !is_signed; -} - -DataTypePtr converted_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const bool nullable = column->max_definition_level() > 0; - switch (column->converted_type()) { - case ::parquet::ConvertedType::UTF8: - case ::parquet::ConvertedType::ENUM: - case ::parquet::ConvertedType::JSON: - case ::parquet::ConvertedType::BSON: - return create_type(TYPE_STRING, nullable); - case ::parquet::ConvertedType::DECIMAL: - mark_decimal(column, column->type_precision(), column->type_scale(), result); - return create_type(decimal_primitive_type(column->type_precision()), nullable, - column->type_precision(), column->type_scale()); - case ::parquet::ConvertedType::DATE: - return create_type(TYPE_DATEV2, nullable); - case ::parquet::ConvertedType::TIME_MILLIS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIME_MICROS: - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - case ::parquet::ConvertedType::TIMESTAMP_MILLIS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 3); - case ::parquet::ConvertedType::TIMESTAMP_MICROS: - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = true; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - return create_type(TYPE_DATETIMEV2, nullable, 0, 6); - // Parquet stores signed and unsigned integer logical annotations on signed physical carriers: - // INT_8/UINT_8/INT_16/UINT_16/INT_32/UINT_32 use physical INT32, and - // INT_64/UINT_64 use physical INT64. Doris maps unsigned integers to the next wider - // signed type so all values in the unsigned range can be represented. - case ::parquet::ConvertedType::INT_8: - mark_integer(8, true, result); - return create_type(TYPE_TINYINT, nullable); - case ::parquet::ConvertedType::UINT_8: - mark_integer(8, false, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::INT_16: - mark_integer(16, true, result); - return create_type(TYPE_SMALLINT, nullable); - case ::parquet::ConvertedType::UINT_16: - mark_integer(16, false, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::INT_32: - mark_integer(32, true, result); - return create_type(TYPE_INT, nullable); - case ::parquet::ConvertedType::UINT_32: - mark_integer(32, false, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::INT_64: - mark_integer(64, true, result); - return create_type(TYPE_BIGINT, nullable); - case ::parquet::ConvertedType::UINT_64: - mark_integer(64, false, result); - return create_type(TYPE_LARGEINT, nullable); - case ::parquet::ConvertedType::NONE: - default: - return nullptr; - } -} - -DataTypePtr logical_type_to_doris_type(const ::parquet::ColumnDescriptor* column, - ParquetTypeDescriptor* result) { - const auto& logical_type = column->logical_type(); - if (logical_type == nullptr || !logical_type->is_valid() || logical_type->is_none()) { - return nullptr; - } - const bool nullable = column->max_definition_level() > 0; - if (logical_type->is_string() || logical_type->is_enum() || logical_type->is_JSON() || - logical_type->is_BSON() || logical_type->is_UUID()) { - return create_type(TYPE_STRING, nullable); - } - if (logical_type->is_decimal()) { - const auto& decimal_type = static_cast(*logical_type); - mark_decimal(column, decimal_type.precision(), decimal_type.scale(), result); - return create_type(decimal_primitive_type(decimal_type.precision()), nullable, - decimal_type.precision(), decimal_type.scale()); - } - if (logical_type->is_date()) { - return create_type(TYPE_DATEV2, nullable); - } - if (logical_type->is_time()) { - const auto& time_type = static_cast(*logical_type); - if (time_type.is_adjusted_to_utc()) { - result->unsupported_reason = "Parquet TIME with isAdjustedToUTC=true is not supported"; - return nullptr; - } - int scale = 0; - if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (time_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else { - return nullptr; - } - return create_type(TYPE_TIMEV2, nullable, 0, scale); - } - if (logical_type->is_timestamp()) { - const auto& timestamp_type = - static_cast(*logical_type); - int scale = 0; - if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MILLIS) { - scale = 3; - result->time_unit = ParquetTimeUnit::MILLIS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::MICROS) { - scale = 6; - result->time_unit = ParquetTimeUnit::MICROS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_MICROS; - } else if (timestamp_type.time_unit() == ::parquet::LogicalType::TimeUnit::NANOS) { - scale = 6; - result->time_unit = ParquetTimeUnit::NANOS; - result->extra_type_info = ParquetExtraTypeInfo::UNIT_NS; - } else { - return nullptr; - } - result->is_timestamp = true; - result->timestamp_is_adjusted_to_utc = timestamp_type.is_adjusted_to_utc(); - return create_type(TYPE_DATETIMEV2, nullable, 0, scale); - } - if (logical_type->is_int()) { - const auto& int_type = static_cast(*logical_type); - mark_integer(int_type.bit_width(), int_type.is_signed(), result); - switch (int_type.bit_width()) { - case 8: - return create_type(int_type.is_signed() ? TYPE_TINYINT : TYPE_SMALLINT, nullable); - case 16: - return create_type(int_type.is_signed() ? TYPE_SMALLINT : TYPE_INT, nullable); - case 32: - return create_type(int_type.is_signed() ? TYPE_INT : TYPE_BIGINT, nullable); - case 64: - return create_type(int_type.is_signed() ? TYPE_BIGINT : TYPE_LARGEINT, nullable); - default: - return nullptr; - } - } - if (logical_type->is_float16()) { - if (column->physical_type() != ::parquet::Type::FIXED_LEN_BYTE_ARRAY || - column->type_length() != 2) { - return nullptr; - } - result->extra_type_info = ParquetExtraTypeInfo::FLOAT16; - return create_type(TYPE_FLOAT, nullable); - } - return nullptr; -} - -DataTypePtr physical_type_to_doris_type(const ::parquet::ColumnDescriptor* column) { - const bool nullable = column->max_definition_level() > 0; - DataTypePtr type; - switch (column->physical_type()) { - case ::parquet::Type::BOOLEAN: - type = std::make_shared(); - break; - case ::parquet::Type::INT32: - type = std::make_shared(); - break; - case ::parquet::Type::INT64: - type = std::make_shared(); - break; - case ::parquet::Type::FLOAT: - type = std::make_shared(); - break; - case ::parquet::Type::DOUBLE: - type = std::make_shared(); - break; - case ::parquet::Type::BYTE_ARRAY: - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: - type = std::make_shared(); - break; - case ::parquet::Type::INT96: - type = create_type(TYPE_DATETIMEV2, nullable, 0, 6); - break; - default: - return nullptr; - } - return nullable ? make_nullable(type) : type; -} - -} // namespace - -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column) { - if (column == nullptr) { - return {}; - } - auto path = column->path(); - if (path) { - return path->ToDotString(); - } - return column->name(); -} - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column) { - ParquetTypeDescriptor result; - if (column == nullptr) { - return result; - } - - result.physical_type = column->physical_type(); - result.converted_type = column->converted_type(); - result.fixed_length = column->type_length(); - result.physical_doris_type = physical_type_to_doris_type(column); - - if (auto logical_type = logical_type_to_doris_type(column, &result); logical_type != nullptr) { - result.doris_type = logical_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - } else if (auto converted_type = converted_type_to_doris_type(column, &result); - converted_type != nullptr) { - result.doris_type = converted_type; - } else if (!result.unsupported_reason.empty()) { - result.doris_type = nullptr; - } else { - result.doris_type = result.physical_doris_type; - if (result.physical_type == ::parquet::Type::INT96) { - result.extra_type_info = ParquetExtraTypeInfo::IMPALA_TIMESTAMP; - } - } - - result.is_string_like = !result.is_decimal && - result.extra_type_info != ParquetExtraTypeInfo::FLOAT16 && - (result.physical_type == ::parquet::Type::BYTE_ARRAY || - result.physical_type == ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - - return result; -} DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor) { switch (type_descriptor.physical_type) { - case ::parquet::Type::BOOLEAN: + case tparquet::Type::BOOLEAN: return DecodedValueKind::BOOL; - case ::parquet::Type::INT32: + case tparquet::Type::INT32: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 32) { return DecodedValueKind::UINT32; } return DecodedValueKind::INT32; - case ::parquet::Type::INT64: + case tparquet::Type::INT64: if (type_descriptor.is_unsigned_integer && type_descriptor.integer_bit_width == 64) { return DecodedValueKind::UINT64; } return DecodedValueKind::INT64; - case ::parquet::Type::INT96: + case tparquet::Type::INT96: return DecodedValueKind::INT96; - case ::parquet::Type::FLOAT: + case tparquet::Type::FLOAT: return DecodedValueKind::FLOAT; - case ::parquet::Type::DOUBLE: + case tparquet::Type::DOUBLE: return DecodedValueKind::DOUBLE; - case ::parquet::Type::FIXED_LEN_BYTE_ARRAY: + case tparquet::Type::FIXED_LEN_BYTE_ARRAY: return DecodedValueKind::FIXED_BINARY; - case ::parquet::Type::BYTE_ARRAY: + case tparquet::Type::BYTE_ARRAY: default: return DecodedValueKind::BINARY; } diff --git a/be/src/format_v2/parquet/parquet_type.h b/be/src/format_v2/parquet/parquet_type.h index feded03ccf25fc..7ab3d2b3b39d8d 100644 --- a/be/src/format_v2/parquet/parquet_type.h +++ b/be/src/format_v2/parquet/parquet_type.h @@ -15,17 +15,13 @@ #pragma once -#include +#include #include #include "core/data_type/data_type.h" #include "core/data_type_serde/decoded_column_view.h" -namespace parquet { -class ColumnDescriptor; -} // namespace parquet - namespace doris::format::parquet { // ============================================================================ @@ -53,14 +49,15 @@ enum class ParquetTimeUnit { // ============================================================================ // ============================================================================ struct ParquetTypeDescriptor { + // Keep the V2 semantic tree on generated Thrift enums; using parquet-cpp descriptors here + // would reintroduce a second metadata model and make native planning bypassable. DataTypePtr doris_type; // Physical fallback used only to keep file schema construction alive when the logical type is // unsupported. Column reader creation still rejects unsupported_reason before decoding. DataTypePtr physical_doris_type; ParquetExtraTypeInfo extra_type_info = ParquetExtraTypeInfo::NONE; ParquetTimeUnit time_unit = ParquetTimeUnit::UNKNOWN; - ::parquet::Type::type physical_type = ::parquet::Type::UNDEFINED; - ::parquet::ConvertedType::type converted_type = ::parquet::ConvertedType::UNDEFINED; + tparquet::Type::type physical_type = tparquet::Type::INT32; int integer_bit_width = -1; // bit width for INT_8/16/32/64 int decimal_precision = -1; // precision for DECIMAL(p,s) int decimal_scale = -1; // scale for DECIMAL(p,s) @@ -73,10 +70,6 @@ struct ParquetTypeDescriptor { std::string unsupported_reason; // non-empty when this Parquet logical type is unsupported }; -std::string parquet_column_name(const ::parquet::ColumnDescriptor* column); - -ParquetTypeDescriptor resolve_parquet_type(const ::parquet::ColumnDescriptor* column); - DecodedValueKind decoded_value_kind(const ParquetTypeDescriptor& type_descriptor); } // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp index 7176e1a054e77a..cb54cab86b27d8 100644 --- a/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp +++ b/be/test/format_v2/parquet/parquet_page_cache_range_test.cpp @@ -26,134 +26,6 @@ namespace doris::format::parquet { namespace { -void expect_plan_entry(const ParquetPageCacheReadPlanEntry& entry, - const ParquetPageCacheRange& cached_range, int64_t copy_offset_in_cache, - int64_t output_offset, int64_t copy_size) { - EXPECT_EQ(entry.cached_range.offset, cached_range.offset); - EXPECT_EQ(entry.cached_range.size, cached_range.size); - EXPECT_EQ(entry.copy_offset_in_cache, copy_offset_in_cache); - EXPECT_EQ(entry.output_offset, output_offset); - EXPECT_EQ(entry.copy_size, copy_size); -} - -TEST(ParquetPageCacheRangeTest, SubsetRequestHitsSingleCachedRange) { - const std::vector cached_ranges = { - {100, 100}, - }; - - // Request [120, 150) is fully inside cached [100, 200). The reader should lookup - // the exact cached key [100, 200), then copy from cached offset 20 into output offset 0. - auto plan = detail::plan_page_cache_range_read(120, 30, cached_ranges); - - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestHitsMultipleAdjacentCachedRanges) { - const std::vector cached_ranges = { - {180, 80}, - {100, 80}, - }; - - // Request [100, 260) is larger than either cached entry, but the two cached ranges - // exactly cover it. The copy plan stitches the two exact cache entries together. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 80}, 0, 0, 80); - expect_plan_entry(plan[1], {180, 80}, 0, 80, 80); -} - -TEST(ParquetPageCacheRangeTest, SupersetRequestCanUseOverlappingCachedRanges) { - const std::vector cached_ranges = { - {150, 110}, - {100, 100}, - }; - - // Request [100, 260) is covered by overlapping cached ranges. The first copy uses - // [100, 200); the second resumes at cursor 200 and copies the tail from [150, 260). - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - ASSERT_EQ(plan.size(), 2); - expect_plan_entry(plan[0], {100, 100}, 0, 0, 100); - expect_plan_entry(plan[1], {150, 110}, 50, 100, 60); -} - -TEST(ParquetPageCacheRangeTest, PartialOverlapWithoutFullCoverageMisses) { - const std::vector cached_ranges = { - {100, 80}, - {200, 60}, - }; - - // Cached ranges cover [100, 180) and [200, 260), but [180, 200) is missing. - // The caller must read the whole request from the file instead of returning - // a partially cached result. - auto plan = detail::plan_page_cache_range_read(100, 160, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, NonCoveringAndInvalidRangesAreIgnored) { - const std::vector cached_ranges = { - {50, 20}, {100, 0}, {100, -1}, {180, 20}, {120, 30}, - }; - - // Only [120, 150) intersects the request, but it does not cover the request start - // [100, 120), so this is still a miss. - auto plan = detail::plan_page_cache_range_read(100, 50, cached_ranges); - - EXPECT_TRUE(plan.empty()); -} - -TEST(ParquetPageCacheRangeTest, InvalidRequestMisses) { - const std::vector cached_ranges = { - {100, 100}, - }; - - EXPECT_TRUE(detail::plan_page_cache_range_read(-1, 10, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, 0, cached_ranges).empty()); - EXPECT_TRUE(detail::plan_page_cache_range_read(100, -1, cached_ranges).empty()); -} - -TEST(ParquetPageCacheRangeTest, PerFileRangeIndexDeduplicatesAndEvictsAtCapacity) { - detail::ParquetPageCacheRangeIndex index(3); - index.insert({200, 20}); - index.insert({100, 30}); - index.insert({100, 10}); - index.insert({100, 30}); - - EXPECT_EQ(index.size(), 3); - index.insert({300, 40}); - const auto ranges = index.ranges(); - ASSERT_EQ(ranges.size(), 3); - EXPECT_EQ(ranges[0].offset, 100); - EXPECT_EQ(ranges[0].size, 30); - EXPECT_EQ(ranges[1].offset, 200); - EXPECT_EQ(ranges[1].size, 20); - EXPECT_EQ(ranges[2].offset, 300); - - index.erase({100, 30}); - EXPECT_EQ(index.size(), 2); -} - -TEST(ParquetPageCacheRangeTest, DirectorySharesBoundedIndexAcrossReaderLifetimes) { - detail::ParquetPageCacheRangeDirectory directory(2); - auto first_reader_index = directory.get_or_create("file-a"); - first_reader_index->insert({100, 100}); - first_reader_index.reset(); - - // The directory owns the per-file index, so reader B still discovers reader A's wider cache - // entry and can plan a subset hit after A closes. - auto second_reader_index = directory.get_or_create("file-a"); - const auto plan = detail::plan_page_cache_range_read(120, 30, second_reader_index->ranges()); - ASSERT_EQ(plan.size(), 1); - expect_plan_entry(plan[0], {100, 100}, 20, 0, 30); - - directory.get_or_create("file-b"); - directory.get_or_create("file-c"); - EXPECT_EQ(directory.size(), 2); -} - TEST(ParquetPageCacheRangeTest, ValidPrefetchRangesSkipInvalidAndOverflowRanges) { const std::vector ranges = { {100, 50}, diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 753e4516b38db8..10bdc2d8a935dd 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -985,31 +985,6 @@ void write_dictionary_filter_with_trailing_column_parquet_file(const std::string builder.build())); } -void write_nested_dictionary_filter_parquet_file(const std::string& file_path) { - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make( - schema, {build_struct_array({1, 2, 3, 4, 5, 6}, {"aa", "az", "lm", "lz", "za", "zz"})}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.enable_dictionary("s.name"); - builder.disable_dictionary("s.identifier.field_id"); - builder.disable_statistics(); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1, - builder.build())); -} - void write_dictionary_edge_parquet_file(const std::string& file_path) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), @@ -1035,62 +1010,6 @@ void write_dictionary_edge_parquet_file(const std::string& file_path) { builder.build())); } -void write_nested_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - std::vector names; - names.reserve(ids.size()); - for (const auto id : ids) { - names.push_back("name-" + std::to_string(id)); - } - auto id_field = arrow::field("id", arrow::int32(), false); - auto name_field = arrow::field("name", arrow::utf8(), false); - auto struct_type = arrow::struct_({id_field, name_field}); - auto schema = arrow::schema({ - arrow::field("s", struct_type, false), - }); - auto table = arrow::Table::Make(schema, {build_struct_array(ids, names)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - -void write_page_index_filter_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false), - }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); - - auto file_result = arrow::io::FileOutputStream::Open(file_path); - ASSERT_TRUE(file_result.ok()) << file_result.status(); - std::shared_ptr out = *file_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); - builder.disable_dictionary(); - builder.enable_write_page_index(); - builder.write_batch_size(8); - builder.data_pagesize(10); - PARQUET_THROW_NOT_OK(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - ids.size(), builder.build())); -} - void write_page_index_filter_pair_parquet_file(const std::string& file_path) { std::vector ids(128); std::iota(ids.begin(), ids.end(), 0); @@ -2560,41 +2479,6 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByStatistics) { TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto value_chunk = row_group->ColumnChunk(1); - ASSERT_NE(value_chunk, nullptr); - ASSERT_TRUE(value_chunk->has_dictionary_page()); - ASSERT_TRUE(value_chunk->statistics() == nullptr || - !value_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 2); - - format::FileScanRequest plan_request; - plan_request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - plan_request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - plan_request, scan_range, false, &plan) - .ok()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 5); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 5); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, 1); - auto reader = create_reader(); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -2904,205 +2788,6 @@ TEST_F(NewParquetReaderTest, DictionaryPredicateKeepsNestedOrResidualConjunct) { EXPECT_EQ(profile.get_counter("SelectedRows")->value(), 1); } -TEST_F(NewParquetReaderTest, ScanRangeFiltersRowGroupsBeforeDictionaryPruning) { - write_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(1), format::LocalIndex(1)); - request.conjuncts.push_back(create_string_in_conjunct(1, {"lm"})); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 2); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByStatistics) { - write_struct_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 2); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 2); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotFilterRowGroupsByDictionary) { - write_nested_dictionary_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 6); - for (int row_group_idx = 0; row_group_idx < 6; ++row_group_idx) { - auto row_group = parquet_file_reader->metadata()->RowGroup(row_group_idx); - ASSERT_NE(row_group, nullptr); - auto name_chunk = row_group->ColumnChunk(1); - ASSERT_NE(name_chunk, nullptr); - ASSERT_TRUE(name_chunk->has_dictionary_page()); - ASSERT_TRUE(name_chunk->statistics() == nullptr || !name_chunk->statistics()->HasMinMax()); - } - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[1]->name, "name"); - - format::FileScanRequest request; - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 6); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 6); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_dictionary, 0); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 0); -} - -TEST_F(NewParquetReaderTest, PlannerNarrowsRowRangesByPageIndex) { - write_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - - format::FileScanRequest request; - request.predicate_columns = {field_projection(0)}; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_GT(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_LT(plan.row_groups[0].selected_ranges.front().length, 128); - auto skip_plan_it = plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(skip_plan_it, plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(skip_plan_it->second.leaf_column_id, 0); - EXPECT_GT(skip_plan_it->second.skipped_ranges.size(), 0); - EXPECT_GT(skip_plan_it->second.skipped_pages.size(), 1); - ASSERT_EQ(skip_plan_it->second.skipped_pages.size(), - skip_plan_it->second.skipped_page_compressed_sizes.size()); - int64_t skipped_compressed_bytes = 0; - for (size_t page_idx = 0; page_idx < skip_plan_it->second.skipped_pages.size(); ++page_idx) { - if (skip_plan_it->second.should_skip_page(page_idx)) { - skipped_compressed_bytes += skip_plan_it->second.skipped_page_compressed_size(page_idx); - } - } - EXPECT_GT(skipped_compressed_bytes, 0); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_GT(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - -TEST_F(NewParquetReaderTest, NestedStructPredicateDoesNotNarrowRowRangesByPageIndex) { - write_nested_page_index_filter_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto page_index_reader = parquet_file_reader->GetPageIndexReader(); - ASSERT_NE(page_index_reader, nullptr); - auto row_group_index_reader = page_index_reader->RowGroup(0); - ASSERT_NE(row_group_index_reader, nullptr); - auto offset_index = row_group_index_reader->GetOffsetIndex(0); - ASSERT_NE(offset_index, nullptr); - ASSERT_GT(offset_index->page_locations().size(), 1); - - std::vector> file_schema; - auto schema_descriptor = parquet_file_reader->metadata()->schema(); - ASSERT_NE(schema_descriptor, nullptr); - ASSERT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - ASSERT_EQ(file_schema.size(), 1); - ASSERT_EQ(file_schema[0]->children.size(), 2); - ASSERT_EQ(file_schema[0]->children[0]->name, "id"); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_greater_than_conjunct(0, 63)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - ASSERT_FALSE(plan.row_groups[0].selected_ranges.empty()); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges.front().length, - parquet_file_reader->metadata()->RowGroup(0)->num_rows()); - EXPECT_TRUE(plan.row_groups[0].page_skip_plans.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 0); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(plan.pruning_stats.selected_row_ranges, plan.row_groups[0].selected_ranges.size()); -} - // Scenario: the selected range starts after page-index-pruned rows. The scheduler defers that range // gap for the non-predicate payload reader, then flushes it exactly once before materialization. The // native RowRanges/OffsetIndex plan advances both readers without decoding the rejected pages or diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 8ab62ccb8275d7..b5ef381dbe1b10 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -468,17 +468,6 @@ void write_page_index_parquet_file(const std::string& file_path) { write_table(file_path, table, ids.size(), false, true); } -void write_page_index_pair_parquet_file(const std::string& file_path) { - std::vector ids(128); - std::iota(ids.begin(), ids.end(), 0); - auto schema = arrow::schema({ - arrow::field("id", arrow::int32(), false), - arrow::field("score", arrow::int32(), false), - }); - auto table = arrow::Table::Make(schema, {build_int32_array(ids), build_int32_array(ids)}); - write_table(file_path, table, ids.size(), false, true); -} - int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { return column_metadata.has_dictionary_page() ? static_cast(column_metadata.dictionary_page_offset()) @@ -524,24 +513,6 @@ void use_schema_order_positions(format::FileScanRequest* request, } } -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - auto schema_descriptor = reader.metadata()->schema(); - EXPECT_NE(schema_descriptor, nullptr); - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*schema_descriptor, &file_schema).ok()); - return file_schema; -} - -int64_t count_range_rows(const std::vector& ranges) { - int64_t rows = 0; - for (const auto& range : ranges) { - rows += range.length; - } - return rows; -} - class ParquetScanTest : public testing::Test { protected: void SetUp() override { @@ -642,89 +613,6 @@ TEST(ParquetScanSelectionTest, NativeLazySkipBitmapIsBounded) { MAX_NATIVE_LAZY_SKIP_ROWS); } -TEST_F(ParquetScanTest, PlanRowGroupsAppliesScanRangeBeforeStatistics) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - const auto [range_start_offset, range_size] = row_group_mid_range(_file_path, 1); - format::parquet::ParquetScanRange scan_range; - scan_range.start_offset = range_start_offset; - scan_range.size = range_size; - scan_range.file_size = static_cast(std::filesystem::file_size(_file_path)); - - format::parquet::RowGroupScanPlan plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 1); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 2); -} - -TEST_F(ParquetScanTest, PlanRowGroupsPreservesFirstFileRowAcrossPrunedRowGroups) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 5)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - ASSERT_EQ(plan.row_groups.size(), 1); - EXPECT_EQ(plan.row_groups[0].row_group_id, 2); - EXPECT_EQ(plan.row_groups[0].first_file_row, 4); - EXPECT_EQ(plan.row_groups[0].row_group_rows, 2); - ASSERT_EQ(plan.row_groups[0].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[0].selected_ranges[0].length, 2); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 2); - EXPECT_EQ(plan.pruning_stats.filtered_group_rows, 4); -} - -TEST_F(ParquetScanTest, PlanRowGroupsSelectsAllRowGroupsWithoutFilters) { - write_int_pair_parquet_file(_file_path, 2); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 3); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - - ASSERT_EQ(plan.row_groups.size(), 3); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 3); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 3); - for (size_t row_group_idx = 0; row_group_idx < plan.row_groups.size(); ++row_group_idx) { - EXPECT_EQ(plan.row_groups[row_group_idx].row_group_id, row_group_idx); - EXPECT_EQ(plan.row_groups[row_group_idx].first_file_row, - static_cast(row_group_idx * 2)); - ASSERT_EQ(plan.row_groups[row_group_idx].selected_ranges.size(), 1); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].start, 0); - EXPECT_EQ(plan.row_groups[row_group_idx].selected_ranges[0].length, 2); - EXPECT_TRUE(plan.row_groups[row_group_idx].page_skip_plans.empty()); - } -} - TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater) { format::parquet::RowGroupScanPlan plan; plan.row_groups.push_back( @@ -748,131 +636,6 @@ TEST(ParquetScanConditionCacheTest, HitKeepsCachedBaseWhenCurrentPlanStartsLater EXPECT_EQ(ctx->base_granule, 0); } -TEST_F(ParquetScanTest, PageIndexIntersectsMultipleFiltersAndBuildsSkipPlan) { - write_page_index_pair_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest single_filter_request; - format::FileScanRequestBuilder single_filter_builder(&single_filter_request); - ASSERT_TRUE(single_filter_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - single_filter_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - format::parquet::RowGroupScanPlan single_filter_plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - single_filter_request, scan_range, false, &single_filter_plan) - .ok()); - ASSERT_EQ(single_filter_plan.row_groups.size(), 1); - const int64_t single_filter_rows = - count_range_rows(single_filter_plan.row_groups[0].selected_ranges); - - format::FileScanRequest intersect_request; - format::FileScanRequestBuilder intersect_builder(&intersect_request); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(0)).ok()); - ASSERT_TRUE(intersect_builder.add_predicate_column(format::LocalColumnId(1)).ok()); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - intersect_request.conjuncts.push_back( - create_int32_zonemap_conjunct(1, Int32ZoneMapExpr::Op::LT, 96)); - format::parquet::RowGroupScanPlan intersect_plan; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups( - *parquet_file_reader->metadata(), parquet_file_reader.get(), file_schema, - intersect_request, scan_range, false, &intersect_plan) - .ok()); - ASSERT_EQ(intersect_plan.row_groups.size(), 1); - ASSERT_FALSE(intersect_plan.row_groups[0].selected_ranges.empty()); - const int64_t intersect_rows = count_range_rows(intersect_plan.row_groups[0].selected_ranges); - EXPECT_GT(single_filter_rows, intersect_rows); - EXPECT_GT(intersect_plan.row_groups[0].selected_ranges.front().start, 0); - const auto& last_range = intersect_plan.row_groups[0].selected_ranges.back(); - EXPECT_LT(last_range.start + last_range.length, 128); - EXPECT_GT(intersect_plan.pruning_stats.filtered_page_rows, 0); - EXPECT_EQ(intersect_plan.pruning_stats.selected_row_ranges, - intersect_plan.row_groups[0].selected_ranges.size()); - - auto id_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(0); - ASSERT_NE(id_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(id_skip_plan->second.leaf_column_id, 0); - EXPECT_FALSE(id_skip_plan->second.empty()); - auto score_skip_plan = intersect_plan.row_groups[0].page_skip_plans.find(1); - ASSERT_NE(score_skip_plan, intersect_plan.row_groups[0].page_skip_plans.end()); - EXPECT_EQ(score_skip_plan->second.leaf_column_id, 1); - EXPECT_FALSE(score_skip_plan->second.empty()); -} - -TEST_F(ParquetScanTest, PageIndexCanFullyFilterRowGroupAfterRangeIntersection) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - ASSERT_EQ(parquet_file_reader->metadata()->num_row_groups(), 1); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GE, 32)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::LT, 32)); - - format::parquet::RowGroupScanPlan plan; - format::parquet::ParquetScanRange scan_range; - ASSERT_TRUE(format::parquet::plan_parquet_row_groups(*parquet_file_reader->metadata(), - parquet_file_reader.get(), file_schema, - request, scan_range, false, &plan) - .ok()); - EXPECT_TRUE(plan.row_groups.empty()); - EXPECT_EQ(plan.pruning_stats.total_row_groups, 1); - EXPECT_EQ(plan.pruning_stats.selected_row_groups, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(plan.pruning_stats.filtered_row_groups_by_page_index, 1); - EXPECT_EQ(plan.pruning_stats.filtered_page_rows, 128); -} - -TEST_F(ParquetScanTest, PageIndexFullRangeWhenDisabledOrUnavailable) { - write_page_index_parquet_file(_file_path); - auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto file_schema = build_file_schema(*parquet_file_reader); - - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 63)); - - const bool old_enable_page_index = config::enable_parquet_page_index; - config::enable_parquet_page_index = false; - std::vector selected_ranges; - std::map page_skip_plans; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - parquet_file_reader.get(), file_schema, request, 0, 128, &selected_ranges, - &page_skip_plans, &pruning_stats) - .ok()); - config::enable_parquet_page_index = old_enable_page_index; - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 128); - EXPECT_TRUE(page_skip_plans.empty()); - EXPECT_EQ(pruning_stats.page_index_read_calls, 0); - - write_int_pair_parquet_file(_file_path, 6); - auto no_index_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); - auto no_index_schema = build_file_schema(*no_index_reader); - format::FileScanRequest no_index_request; - no_index_request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - no_index_request.conjuncts.push_back( - create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 3)); - selected_ranges.clear(); - page_skip_plans.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_group_ranges_by_page_index( - no_index_reader.get(), no_index_schema, no_index_request, 0, 6, - &selected_ranges, &page_skip_plans, &pruning_stats) - .ok()); - ASSERT_EQ(selected_ranges.size(), 1); - EXPECT_EQ(selected_ranges[0].start, 0); - EXPECT_EQ(selected_ranges[0].length, 6); - EXPECT_TRUE(page_skip_plans.empty()); -} - TEST_F(ParquetScanTest, AggregateCountAndMinMaxUseAllSelectedRowGroups) { write_int_pair_parquet_file(_file_path); auto reader = create_reader(); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 98cf963b30080a..4aa0be0fc5adc8 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -16,7 +16,6 @@ // under the License. #include -#include #include #include @@ -34,54 +33,6 @@ #include "format_v2/parquet/parquet_file_context.h" namespace doris::format::parquet { -namespace { - -std::vector> build_fields( - const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - EXPECT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - return fields; -} - -Status build_status(const std::vector<::parquet::schema::NodePtr>& nodes) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, nodes); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - std::vector> fields; - return build_parquet_column_schema(descriptor, &fields); -} - -} // namespace - -TEST(ParquetSchemaTest, PrimitiveStateAndFieldIdArePreserved) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("required_i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("optional_i64", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT64, - ::parquet::ConvertedType::NONE, -1, -1, -1, 42), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->local_id, 0); - EXPECT_EQ(fields[0]->name, "required_i32"); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->leaf_column_id, 0); - EXPECT_EQ(fields[0]->nullable_definition_level, 0); - EXPECT_FALSE(fields[0]->type->is_nullable()); - - EXPECT_EQ(fields[1]->local_id, 1); - EXPECT_EQ(fields[1]->parquet_field_id, 42); - EXPECT_EQ(fields[1]->leaf_column_id, 1); - EXPECT_EQ(fields[1]->nullable_definition_level, 1); - EXPECT_TRUE(fields[1]->type->is_nullable()); -} - TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { tparquet::SchemaElement root; root.__set_name("schema"); @@ -124,461 +75,6 @@ TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); ASSERT_NE(mapping->child("minReaderVersion"), nullptr); } - -TEST(ParquetSchemaTest, PrimitiveTypeDescriptorCoversLogicalConvertedAndPhysicalFallback) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(false, - ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64), - ::parquet::schema::PrimitiveNode::Make("i8", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8), - ::parquet::schema::PrimitiveNode::Make("plain", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - }); - - ASSERT_EQ(fields.size(), 3); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_DATETIMEV2); - EXPECT_EQ(fields[0]->type_descriptor.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(fields[0]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - EXPECT_TRUE(fields[0]->type_descriptor.is_timestamp); - EXPECT_FALSE(fields[0]->type_descriptor.timestamp_is_adjusted_to_utc); - - EXPECT_EQ(remove_nullable(fields[1]->type)->get_primitive_type(), TYPE_TINYINT); - EXPECT_EQ(fields[1]->type_descriptor.integer_bit_width, 8); - EXPECT_FALSE(fields[1]->type_descriptor.is_unsigned_integer); - - EXPECT_EQ(remove_nullable(fields[2]->type)->get_primitive_type(), TYPE_DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.physical_type, ::parquet::Type::DOUBLE); - EXPECT_EQ(fields[2]->type_descriptor.extra_type_info, ParquetExtraTypeInfo::NONE); -} - -TEST(ParquetSchemaTest, StructMakesDataTypeChildrenNullableAndPropagatesLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::PrimitiveNode::Make("a", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - })}); - - ASSERT_EQ(fields.size(), 1); - const auto& struct_schema = *fields[0]; - EXPECT_EQ(struct_schema.kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(struct_schema.nullable_definition_level, 1); - ASSERT_EQ(struct_schema.children.size(), 2); - EXPECT_EQ(struct_schema.children[0]->definition_level, 1); - EXPECT_EQ(struct_schema.children[1]->definition_level, 2); - EXPECT_EQ(struct_schema.max_definition_level, 2); - - const auto& struct_type = - assert_cast(*remove_nullable(struct_schema.type)); - ASSERT_EQ(struct_type.get_elements().size(), 2); - EXPECT_TRUE(struct_type.get_elements()[0]->is_nullable()); - EXPECT_TRUE(struct_type.get_elements()[1]->is_nullable()); -} - -TEST(ParquetSchemaTest, ListCompatibilityRulesAndLevels) { - const auto standard_list = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - const auto structural_array = ::parquet::schema::GroupNode::Make( - "ys", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "array", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({standard_list, structural_array}); - ASSERT_EQ(fields.size(), 2); - - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(xs.definition_level, 2); - EXPECT_EQ(xs.repetition_level, 1); - ASSERT_EQ(xs.children.size(), 1); - EXPECT_EQ(xs.children[0]->name, "element"); - EXPECT_EQ(xs.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_TRUE(xs.children[0]->type->is_nullable()); - const auto& xs_type = assert_cast(*remove_nullable(xs.type)); - EXPECT_TRUE(xs_type.get_nested_type()->is_nullable()); - - const auto& ys = *fields[1]; - EXPECT_EQ(ys.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(ys.children.size(), 1); - EXPECT_EQ(ys.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(remove_nullable(ys.children[0]->type)->get_primitive_type(), TYPE_STRUCT); -} - -TEST(ParquetSchemaTest, LegacyListElementResolutionRulesArePreserved) { - const auto two_level_list = ::parquet::schema::GroupNode::Make( - "two_level", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - const auto tuple_list = ::parquet::schema::GroupNode::Make( - "tuple_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "tuple_list_tuple", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64)})}, - ::parquet::ConvertedType::LIST); - const auto multi_field_list = ::parquet::schema::GroupNode::Make( - "records", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - ::parquet::schema::PrimitiveNode::Make("name", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::LIST); - const auto fields = build_fields({two_level_list, tuple_list, multi_field_list}); - ASSERT_EQ(fields.size(), 3); - - const auto& two_level = *fields[0]; - EXPECT_EQ(two_level.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(two_level.definition_level, 2); - EXPECT_EQ(two_level.repetition_level, 1); - ASSERT_EQ(two_level.children.size(), 1); - EXPECT_EQ(two_level.children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(two_level.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(two_level.children[0]->type)->get_primitive_type(), TYPE_INT); - - const auto& tuple = *fields[1]; - ASSERT_EQ(tuple.children.size(), 1); - EXPECT_EQ(tuple.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(tuple.children[0]->name, "element"); - ASSERT_EQ(tuple.children[0]->children.size(), 1); - EXPECT_EQ(tuple.children[0]->children[0]->name, "value"); - - const auto& multi_field = *fields[2]; - ASSERT_EQ(multi_field.children.size(), 1); - EXPECT_EQ(multi_field.children[0]->kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(multi_field.children[0]->children.size(), 2); - EXPECT_EQ(multi_field.children[0]->children[0]->name, "id"); - EXPECT_EQ(multi_field.children[0]->children[1]->name, "name"); -} - -TEST(ParquetSchemaTest, NestedRepeatedInsideListElementIsWrappedOnce) { - const auto list_with_repeated_child = ::parquet::schema::GroupNode::Make( - "outer", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({list_with_repeated_child}); - ASSERT_EQ(fields.size(), 1); - const auto& outer = *fields[0]; - EXPECT_EQ(outer.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(outer.children.size(), 1); - const auto& element = *outer.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::STRUCT); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.children[0]->name, "items"); - ASSERT_EQ(element.children[0]->children.size(), 1); - EXPECT_EQ(element.children[0]->children[0]->name, "element"); -} - -TEST(ParquetSchemaTest, ListWrapperWithLogicalAnnotationIsPreservedAsElement) { - const auto annotated_repeated_group = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - EXPECT_FALSE(build_status({annotated_repeated_group}).ok()); - - const auto nested_list_wrapper = ::parquet::schema::GroupNode::Make( - "xs", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST)}, - ::parquet::ConvertedType::LIST); - - const auto fields = build_fields({nested_list_wrapper}); - ASSERT_EQ(fields.size(), 1); - const auto& xs = *fields[0]; - EXPECT_EQ(xs.kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(xs.children.size(), 1); - const auto& element = *xs.children[0]; - EXPECT_EQ(element.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(element.name, "element"); - ASSERT_EQ(element.children.size(), 1); - EXPECT_EQ(element.children[0]->name, "element"); - EXPECT_EQ(remove_nullable(element.children[0]->type)->get_primitive_type(), TYPE_INT); -} - -TEST(ParquetSchemaTest, MapWrapperIsFoldedAndOptionalKeyIsAllowed) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); - - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_EQ(map_schema.kind, ParquetColumnSchemaKind::MAP); - EXPECT_EQ(map_schema.definition_level, 2); - EXPECT_EQ(map_schema.repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->name, "key"); - EXPECT_EQ(map_schema.children[1]->name, "value"); - EXPECT_TRUE(map_schema.children[0]->type->is_nullable()); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); -} - -TEST(ParquetSchemaTest, StandardMapLevelsAndDataTypesAreBuiltFromEntryContext) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "m", ::parquet::Repetition::REQUIRED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - { - ::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32), - })}, - ::parquet::ConvertedType::MAP)}); - - ASSERT_EQ(fields.size(), 1); - const auto& map_schema = *fields[0]; - EXPECT_FALSE(map_schema.type->is_nullable()); - EXPECT_EQ(map_schema.definition_level, 1); - EXPECT_EQ(map_schema.repetition_level, 1); - EXPECT_EQ(map_schema.repeated_repetition_level, 1); - EXPECT_EQ(map_schema.max_definition_level, 2); - EXPECT_EQ(map_schema.max_repetition_level, 1); - ASSERT_EQ(map_schema.children.size(), 2); - EXPECT_EQ(map_schema.children[0]->definition_level, 1); - EXPECT_EQ(map_schema.children[0]->repetition_level, 1); - EXPECT_EQ(map_schema.children[1]->definition_level, 2); - EXPECT_EQ(map_schema.children[1]->nullable_definition_level, 2); - - const auto& map_type = assert_cast(*remove_nullable(map_schema.type)); - EXPECT_TRUE(map_type.get_key_type()->is_nullable()); - EXPECT_TRUE(map_type.get_value_type()->is_nullable()); -} - -TEST(ParquetSchemaTest, BareRepeatedFieldsAreWrappedAsLists) { - const auto fields = build_fields({ - ::parquet::schema::PrimitiveNode::Make("items", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32), - ::parquet::schema::GroupNode::Make( - "links", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("url", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("rank", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }); - - ASSERT_EQ(fields.size(), 2); - EXPECT_EQ(fields[0]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[0]->children.size(), 1); - EXPECT_EQ(fields[0]->children[0]->kind, ParquetColumnSchemaKind::PRIMITIVE); - EXPECT_EQ(fields[0]->children[0]->name, "element"); - - EXPECT_EQ(fields[1]->kind, ParquetColumnSchemaKind::LIST); - ASSERT_EQ(fields[1]->children.size(), 1); - EXPECT_EQ(fields[1]->children[0]->kind, ParquetColumnSchemaKind::STRUCT); - EXPECT_EQ(fields[1]->children[0]->name, "element"); -} - -TEST(ParquetSchemaTest, DeepLevelChainPropagatesDefinitionAndRepetitionLevels) { - const auto fields = build_fields({::parquet::schema::GroupNode::Make( - "s", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "inner", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make( - "items", ::parquet::Repetition::REPEATED, ::parquet::Type::INT32)})})}); - - ASSERT_EQ(fields.size(), 1); - const auto& s = *fields[0]; - EXPECT_EQ(s.definition_level, 1); - EXPECT_EQ(s.nullable_definition_level, 1); - ASSERT_EQ(s.children.size(), 1); - const auto& inner = *s.children[0]; - EXPECT_EQ(inner.definition_level, 2); - EXPECT_EQ(inner.nullable_definition_level, 2); - ASSERT_EQ(inner.children.size(), 1); - const auto& items = *inner.children[0]; - EXPECT_EQ(items.kind, ParquetColumnSchemaKind::LIST); - EXPECT_EQ(items.definition_level, 3); - EXPECT_EQ(items.repetition_level, 1); - EXPECT_EQ(items.repeated_ancestor_definition_level, 3); - EXPECT_EQ(items.repeated_repetition_level, 1); - EXPECT_EQ(items.max_definition_level, 3); - EXPECT_EQ(items.max_repetition_level, 1); - ASSERT_EQ(items.children.size(), 1); - EXPECT_EQ(items.children[0]->definition_level, 3); - EXPECT_EQ(items.children[0]->repetition_level, 1); -} - -TEST(ParquetSchemaTest, BuildEntryValidatesNullPointerAndEmptyRoot) { - auto empty_root = ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, - ::parquet::schema::NodeVector {}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(empty_root); - - EXPECT_FALSE(build_parquet_column_schema(descriptor, nullptr).ok()); - - std::vector> fields; - ASSERT_TRUE(build_parquet_column_schema(descriptor, &fields).ok()); - EXPECT_TRUE(fields.empty()); -} - -TEST(ParquetSchemaTest, RejectInvalidListMapAndPreserveUnsupportedTime) { - const auto bad_list = ::parquet::schema::GroupNode::Make( - "bad_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({bad_list}).ok()); - - const auto bad_map = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("entry", ::parquet::Repetition::REPEATED, - ::parquet::Type::INT32)}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({bad_map}).ok()); - - const auto converted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS); - const auto fields = build_fields({converted_time}); - ASSERT_EQ(fields.size(), 1); - EXPECT_EQ(remove_nullable(fields[0]->type)->get_primitive_type(), TYPE_INT); - EXPECT_NE(fields[0]->type_descriptor.unsupported_reason.find( - "Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); -} - -TEST(ParquetSchemaTest, RejectAdditionalInvalidListAndMapLayouts) { - const auto zero_child_list = ::parquet::schema::GroupNode::Make( - "zero_child_list", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make("list", ::parquet::Repetition::REPEATED, - ::parquet::schema::NodeVector {})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({zero_child_list}).ok()); - - const auto repeated_list = ::parquet::schema::GroupNode::Make( - "repeated_list", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "list", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("item", ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::LIST); - EXPECT_FALSE(build_status({repeated_list}).ok()); - - const auto map_with_two_fields = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - { - ::parquet::schema::GroupNode::Make( - "entry1", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - ::parquet::schema::GroupNode::Make( - "entry2", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make( - "key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make("value", - ::parquet::Repetition::OPTIONAL, - ::parquet::Type::INT32)}), - }, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_with_two_fields}).ok()); - - const auto non_repeated_map_entry = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({non_repeated_map_entry}).ok()); - - const auto map_entry_with_one_child = ::parquet::schema::GroupNode::Make( - "bad_map", ::parquet::Repetition::OPTIONAL, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({map_entry_with_one_child}).ok()); - - const auto repeated_map = ::parquet::schema::GroupNode::Make( - "repeated_map", ::parquet::Repetition::REPEATED, - {::parquet::schema::GroupNode::Make( - "key_value", ::parquet::Repetition::REPEATED, - {::parquet::schema::PrimitiveNode::Make("key", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY, - ::parquet::ConvertedType::UTF8), - ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT32)})}, - ::parquet::ConvertedType::MAP); - EXPECT_FALSE(build_status({repeated_map}).ok()); -} - -TEST(ParquetSchemaTest, LogicalUtcTimeIsPreservedForProjection) { - const auto adjusted_time = ::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32); - const auto supported_value = ::parquet::schema::PrimitiveNode::Make( - "value", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64); - const auto row = ::parquet::schema::GroupNode::Make("row", ::parquet::Repetition::OPTIONAL, - {adjusted_time, supported_value}); - const auto fields = build_fields({row}); - ASSERT_EQ(fields.size(), 1); - ASSERT_EQ(fields[0]->children.size(), 2); - EXPECT_EQ(remove_nullable(fields[0]->children[0]->type)->get_primitive_type(), TYPE_INT); - EXPECT_FALSE(fields[0]->children[0]->type_descriptor.unsupported_reason.empty()); - EXPECT_EQ(remove_nullable(fields[0]->children[1]->type)->get_primitive_type(), TYPE_BIGINT); -} - TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsRejected) { tparquet::SchemaElement root; root.__set_name("schema"); diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index 83537dcc4e012e..afe588478cc390 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -17,15 +17,7 @@ #include "format_v2/parquet/parquet_statistics.h" -#include -#include -#include #include -#include -#include -#include -#include -#include #include #include @@ -38,8 +30,6 @@ #include #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/data_type/data_type_timestamptz.h" #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr.h" @@ -50,10 +40,7 @@ #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/reader/native/block_split_bloom_filter.h" #include "io/fs/file_reader.h" -#include "storage/index/bloom_filter/block_split_bloom_filter.h" -#include "storage/index/zone_map/zonemap_eval_context.h" -#include "storage/index/zone_map/zonemap_filter_result.h" - +#include "util/thrift_util.h" namespace doris { namespace { @@ -87,242 +74,6 @@ class StatisticsMemoryFileReader final : public io::FileReader { io::Path _path; bool _closed = false; }; - -std::shared_ptr finish_array(arrow::ArrayBuilder* builder) { - std::shared_ptr array; - EXPECT_TRUE(builder->Finish(&array).ok()); - return array; -} - -std::shared_ptr int32_array(const std::vector>& values) { - arrow::Int32Builder builder; - for (const auto& value : values) { - if (value.has_value()) { - EXPECT_TRUE(builder.Append(*value).ok()); - } else { - EXPECT_TRUE(builder.AppendNull().ok()); - } - } - return finish_array(&builder); -} - -std::shared_ptr uint32_array(const std::vector& values) { - arrow::UInt32Builder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr float_array(const std::vector& values) { - arrow::FloatBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr double_array(const std::vector& values) { - arrow::DoubleBuilder builder; - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -template -std::string encoded_value(const NativeType& value) { - return {reinterpret_cast(&value), sizeof(value)}; -} - -std::shared_ptr string_array(const std::vector& values) { - arrow::StringBuilder builder; - for (const auto& value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::shared_ptr timestamp_array(const std::vector& values) { - arrow::TimestampBuilder builder(arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), - arrow::default_memory_pool()); - for (const auto value : values) { - EXPECT_TRUE(builder.Append(value).ok()); - } - return finish_array(&builder); -} - -std::unique_ptr<::parquet::ParquetFileReader> make_reader( - const std::shared_ptr& table, int64_t row_group_size, bool enable_dictionary, - bool enable_statistics) { - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - - ::parquet::WriterProperties::Builder builder; - builder.version(::parquet::ParquetVersion::PARQUET_2_6); - builder.compression(::parquet::Compression::UNCOMPRESSED); - if (enable_dictionary) { - builder.enable_dictionary(); - } else { - builder.disable_dictionary(); - } - if (!enable_statistics) { - builder.disable_statistics(); - } - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, - row_group_size, builder.build()) - .ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - return ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); -} - -std::vector> build_file_schema( - const ::parquet::ParquetFileReader& reader) { - std::vector> file_schema; - EXPECT_TRUE( - format::parquet::build_parquet_column_schema(*reader.metadata()->schema(), &file_schema) - .ok()); - return file_schema; -} - -template -class TestColumnIndex final : public ::parquet::TypedColumnIndex { -public: - using NativeType = typename ParquetDType::c_type; - - TestColumnIndex(NativeType min_value, NativeType max_value) - : TestColumnIndex(std::vector {min_value}, - std::vector {max_value}) {} - - TestColumnIndex(std::vector min_values, std::vector max_values) - : _null_pages(min_values.size(), false), - _null_counts(min_values.size(), 0), - _min_values(std::move(min_values)), - _max_values(std::move(max_values)) { - EXPECT_EQ(_min_values.size(), _max_values.size()); - for (size_t page_idx = 0; page_idx < _min_values.size(); ++page_idx) { - _non_null_page_indices.push_back(static_cast(page_idx)); - } - } - - const std::vector& null_pages() const override { return _null_pages; } - const std::vector& encoded_min_values() const override { return _encoded_values; } - const std::vector& encoded_max_values() const override { return _encoded_values; } - ::parquet::BoundaryOrder::type boundary_order() const override { - return ::parquet::BoundaryOrder::Unordered; - } - bool has_null_counts() const override { return true; } - const std::vector& null_counts() const override { return _null_counts; } - const std::vector& non_null_page_indices() const override { - return _non_null_page_indices; - } - const std::vector& min_values() const override { return _min_values; } - const std::vector& max_values() const override { return _max_values; } - -private: - const std::vector _null_pages; - const std::vector _encoded_values; - const std::vector _null_counts; - std::vector _non_null_page_indices; - const std::vector _min_values; - const std::vector _max_values; -}; - -class Int32ZoneMapExpr final : public VExpr { -public: - enum class Op { GE, GT, IS_NULL, IS_NOT_NULL }; - - Int32ZoneMapExpr(int column_id, Op op, int32_t value = 0) - : VExpr(std::make_shared(), false), - _column_id(column_id), - _op(op), - _value(value) {} - - const std::string& expr_name() const override { return _expr_name; } - - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, - ColumnPtr&) const override { - return Status::InternalError("Int32ZoneMapExpr is only used by parquet statistics tests"); - } - - bool can_evaluate_zonemap_filter() const override { return true; } - - void collect_slot_column_ids(std::set& column_ids) const override { - column_ids.insert(_column_id); - } - - ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { - auto zone_map = ctx.zone_map(_column_id); - if (zone_map == nullptr) { - return unsupported_zonemap_filter(ctx); - } - if (_op == Op::IS_NULL) { - return zone_map->has_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (_op == Op::IS_NOT_NULL) { - return zone_map->has_not_null ? ZoneMapFilterResult::kMayMatch - : ZoneMapFilterResult::kNoMatch; - } - if (!zone_map->has_not_null) { - return ZoneMapFilterResult::kNoMatch; - } - const auto literal = Field::create_field(_value); - if (_op == Op::GE) { - return zone_map->max_value < literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; - } - return zone_map->max_value <= literal ? ZoneMapFilterResult::kNoMatch - : ZoneMapFilterResult::kMayMatch; - } - -private: - int _column_id; - Op _op; - int32_t _value; - const std::string _expr_name = "Int32ZoneMapExpr"; -}; - -class StringDictionaryInExpr final : public VExpr { -public: - StringDictionaryInExpr(int column_id, std::vector values) - : VExpr(std::make_shared(), false), - _slot(VSlotRef::create_shared(0, column_id, -1, std::make_shared(), - "c0")) { - _values.reserve(values.size()); - for (auto& value : values) { - _values.emplace_back(Field::create_field(std::move(value))); - } - } - - const std::string& expr_name() const override { return _expr_name; } - - Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, - ColumnPtr&) const override { - return Status::InternalError( - "StringDictionaryInExpr is only used by parquet statistics tests"); - } - - bool can_evaluate_dictionary_filter() const override { return true; } - - ZoneMapFilterResult evaluate_dictionary_filter( - const DictionaryEvalContext& ctx) const override { - return expr_zonemap::eval_in_dictionary(ctx, _slot, false, _values); - } - - void collect_slot_column_ids(std::set& column_ids) const override { - _slot->collect_slot_column_ids(column_ids); - } - -private: - VExprSPtr _slot; - std::vector _values; - const std::string _expr_name = "StringDictionaryInExpr"; -}; - class BloomInExpr final : public VExpr { public: BloomInExpr(int column_id, DataTypePtr data_type, std::vector values) @@ -352,24 +103,6 @@ class BloomInExpr final : public VExpr { std::vector _values; const std::string _expr_name = "BloomInExpr"; }; - -format::FileScanRequest request_with_zonemap_conjunct(std::shared_ptr expr) { - format::FileScanRequest request; - request.predicate_columns.push_back( - format::LocalColumnIndex::top_level(format::LocalColumnId(0))); - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared(std::move(expr))); - return request; -} - -format::FileScanRequest request_with_dictionary_conjunct(std::vector values) { - format::FileScanRequest request; - request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); - request.conjuncts.push_back(VExprContext::create_shared( - std::make_shared(0, std::move(values)))); - return request; -} - VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -382,725 +115,15 @@ format::FileScanRequest request_with_bloom_conjunct(DataTypePtr data_type, request.conjuncts = bloom_conjuncts(std::move(data_type), std::move(values)); return request; } - -void add_bloom_field(segment_v2::BlockSplitBloomFilter* bloom_filter, const Field& value, - PrimitiveType type) { - DORIS_CHECK(bloom_filter != nullptr); - switch (type) { - case TYPE_BOOLEAN: { - const bool typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_INT: { - const int32_t typed_value = value.get(); - bloom_filter->add_bytes(reinterpret_cast(&typed_value), sizeof(typed_value)); - break; - } - case TYPE_STRING: { - const auto& typed_value = value.get(); - bloom_filter->add_bytes(typed_value.data(), typed_value.size()); - break; - } - default: - DORIS_CHECK(false); - } -} - -std::unique_ptr bloom_filter_for_fields( - const std::vector& values, PrimitiveType type) { - auto bloom_filter = std::make_unique(); - EXPECT_TRUE(bloom_filter->init(segment_v2::BloomFilter::MINIMUM_BYTES).ok()); - for (const auto& value : values) { - add_bloom_field(bloom_filter.get(), value, type); - } - return bloom_filter; -} - -BloomFilterEvalContext bloom_context(const DataTypePtr& data_type, - const segment_v2::BloomFilter* bloom_filter) { - BloomFilterEvalContext ctx; - ctx.slots.emplace(0, BloomFilterEvalContext::SlotBloomFilter {.data_type = data_type, - .bloom_filter = bloom_filter}); - return ctx; -} - -std::unique_ptr<::parquet::BlockSplitBloomFilter> parquet_bloom_filter() { - auto bloom_filter = std::make_unique<::parquet::BlockSplitBloomFilter>(); - bloom_filter->Init(::parquet::BlockSplitBloomFilter::kMinimumBloomFilterBytes); - return bloom_filter; -} - format::parquet::ParquetColumnSchema uint32_parquet_bloom_schema() { format::parquet::ParquetColumnSchema column_schema; column_schema.type = std::make_shared(); column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::INT32; + column_schema.type_descriptor.physical_type = tparquet::Type::INT32; column_schema.type_descriptor.integer_bit_width = 32; column_schema.type_descriptor.is_unsigned_integer = true; return column_schema; } - -format::parquet::ParquetColumnSchema fixed_len_string_parquet_bloom_schema(int fixed_length) { - format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); - column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = fixed_length; - column_schema.type_descriptor.is_string_like = true; - return column_schema; -} - -format::parquet::ParquetColumnSchema float16_parquet_bloom_schema() { - format::parquet::ParquetColumnSchema column_schema; - column_schema.type = std::make_shared(); - column_schema.type_descriptor.doris_type = column_schema.type; - column_schema.type_descriptor.physical_type = ::parquet::Type::FIXED_LEN_BYTE_ARRAY; - column_schema.type_descriptor.fixed_length = 2; - column_schema.type_descriptor.extra_type_info = format::parquet::ParquetExtraTypeInfo::FLOAT16; - return column_schema; -} - -TEST(ParquetStatisticsTransformTest, ConvertsMinMaxNullCountUnsignedStringAndTimestamp) { - auto table = arrow::Table::Make( - arrow::schema({ - arrow::field("i", arrow::int32(), true), - arrow::field("u", arrow::uint32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false), - }), - {int32_array({1, std::nullopt, 5}), uint32_array({7, 9, 11}), - string_array({"alpha", "beta", "omega"}), timestamp_array({1000, 2000, 3000})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto row_group = reader->metadata()->RowGroup(0); - - const auto int_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], row_group->ColumnChunk(0)->statistics()); - EXPECT_TRUE(int_stats.has_min_max); - EXPECT_TRUE(int_stats.has_null_count); - EXPECT_TRUE(int_stats.has_null); - EXPECT_TRUE(int_stats.has_not_null); - EXPECT_EQ(int_stats.min_value.get(), 1); - EXPECT_EQ(int_stats.max_value.get(), 5); - - const auto uint_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[1], row_group->ColumnChunk(1)->statistics()); - EXPECT_TRUE(uint_stats.has_min_max); - EXPECT_EQ(uint_stats.min_value.get(), 7); - EXPECT_EQ(uint_stats.max_value.get(), 11); - - const auto string_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], row_group->ColumnChunk(2)->statistics()); - EXPECT_TRUE(string_stats.has_min_max); - EXPECT_EQ(string_stats.min_value.get(), "alpha"); - EXPECT_EQ(string_stats.max_value.get(), "omega"); - - auto utc = cctz::utc_time_zone(); - const auto timestamp_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[3], row_group->ColumnChunk(3)->statistics(), &utc); - EXPECT_TRUE(timestamp_stats.has_min_max); - EXPECT_EQ(timestamp_stats.min_value.get_type(), TYPE_DATETIMEV2); - EXPECT_EQ(timestamp_stats.max_value.get_type(), TYPE_DATETIMEV2); - EXPECT_LT(timestamp_stats.min_value, timestamp_stats.max_value); -} - -TEST(ParquetStatisticsTransformTest, DisablesUtcTimestampMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - // America/New_York moved from UTC-04:00 to UTC-05:00 at 2021-11-07 06:00:00 UTC. - // Both UTC endpoints below map to 01:30 local time, while values inside the interval cover - // 01:00 through 01:59. Endpoint conversion therefore cannot represent the true local range. - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636263900 * MICROS_PER_SECOND, - 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto local_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(local_stats.has_not_null); - EXPECT_FALSE(local_stats.has_min_max); - - auto utc = cctz::utc_time_zone(); - const auto utc_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &utc); - EXPECT_TRUE(utc_stats.has_min_max); - EXPECT_LT(utc_stats.min_value, utc_stats.max_value); -} - -TEST(ParquetStatisticsTransformTest, KeepsTimestampTzMinMaxAcrossDstRollback) { - constexpr int64_t MICROS_PER_SECOND = 1000000; - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {timestamp_array({1636263000 * MICROS_PER_SECOND, 1636266600 * MICROS_PER_SECOND})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - auto statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - - // This is the effective type produced by enable_mapping_timestamp_tz. The physical timestamp - // flags intentionally remain adjusted-to-UTC so decoding can preserve the source semantics. - schema[0]->type = std::make_shared(6); - schema[0]->type_descriptor.doris_type = schema[0]->type; - - cctz::time_zone new_york; - ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - const auto timestamp_tz_stats = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics, &new_york); - EXPECT_TRUE(timestamp_tz_stats.has_min_max); - EXPECT_EQ(timestamp_tz_stats.min_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_EQ(timestamp_tz_stats.max_value.get_type(), TYPE_TIMESTAMPTZ); - EXPECT_LT(timestamp_tz_stats.min_value, timestamp_tz_stats.max_value); -} - -TEST(ParquetStatisticsTransformTest, HandlesMissingStatisticsAndAllNullChunks) { - auto no_stats_table = arrow::Table::Make( - arrow::schema({arrow::field("i", arrow::int32(), true)}), {int32_array({1, 2, 3})}); - auto no_stats_reader = make_reader(no_stats_table, 3, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - auto no_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *no_stats_schema[0], - no_stats_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_FALSE(no_stats.has_min_max); - - auto all_null_table = - arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt})}); - auto all_null_reader = make_reader(all_null_table, 2, false, true); - auto all_null_schema = build_file_schema(*all_null_reader); - auto all_null_stats = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *all_null_schema[0], - all_null_reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics()); - EXPECT_TRUE(all_null_stats.has_null_count); - EXPECT_TRUE(all_null_stats.has_null); - EXPECT_FALSE(all_null_stats.has_not_null); - EXPECT_FALSE(all_null_stats.has_min_max); -} - -TEST(ParquetStatisticsTransformTest, MissingNullCountConservativelyReportsPossibleNulls) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({1, std::nullopt, 3})}); - auto reader = make_reader(table, 3, false, true); - auto schema = build_file_schema(*reader); - auto file_statistics = reader->metadata()->RowGroup(0)->ColumnChunk(0)->statistics(); - auto statistics_without_null_count = ::parquet::MakeStatistics<::parquet::Int32Type>( - reader->metadata()->schema()->Column(0), file_statistics->EncodeMin(), - file_statistics->EncodeMax(), file_statistics->num_values(), 0, 0, true, false, false); - - const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], statistics_without_null_count); - EXPECT_FALSE(statistics.has_null_count); - EXPECT_TRUE(statistics.has_null); - EXPECT_TRUE(statistics.has_not_null); - EXPECT_TRUE(statistics.has_min_max); - EXPECT_EQ(statistics.min_value.get(), 1); - EXPECT_EQ(statistics.max_value.get(), 3); -} - -TEST(ParquetStatisticsTransformTest, InvalidFooterNullCountDisablesPruningStatistics) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({7})}); - auto reader = make_reader(table, 1, false, true); - auto schema = build_file_schema(*reader); - tparquet::Statistics malformed; - malformed.__set_null_count(2); - malformed.__set_min_value(encoded_value(7)); - malformed.__set_max_value(encoded_value(7)); - const auto statistics = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], &malformed, 1); - EXPECT_FALSE(statistics.has_null_count); - EXPECT_FALSE(statistics.has_min_max); -} - -TEST(ParquetBloomFilterPruningTest, NativeBloomLayoutRejectsSubBlockAndHugePayloads) { - using format::parquet::detail::validate_native_bloom_filter_layout; - EXPECT_TRUE(validate_native_bloom_filter_layout(8, 12, 32, 44, 128).ok()); - EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, 2, 14, 128).ok()); - EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, 33, 45, 128).ok()); - EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, std::numeric_limits::max(), -1, - std::numeric_limits::max()) - .ok()); - EXPECT_FALSE(validate_native_bloom_filter_layout(120, 12, 32, 44, 128).ok()); - EXPECT_FALSE(validate_native_bloom_filter_layout(8, 12, 32, 200, 128).ok()); -} - -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const float float_nan = std::numeric_limits::quiet_NaN(); - const float float_max = 2.0F; - auto float_stats = ::parquet::MakeStatistics<::parquet::FloatType>( - schema[0]->descriptor, encoded_value(float_nan), encoded_value(float_max), 2, 0, 0, - true, true, false); - const auto converted_float = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], float_stats); - EXPECT_FALSE(converted_float.has_min_max); - EXPECT_TRUE(converted_float.has_not_null); - - const double double_nan = std::numeric_limits::quiet_NaN(); - const double double_min = 1.0; - auto double_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_nan), 2, 0, 0, - true, true, false); - const auto converted_double = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - double_stats); - EXPECT_FALSE(converted_double.has_min_max); - EXPECT_TRUE(converted_double.has_not_null); - - const double double_max = 2.0; - auto finite_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[1]->descriptor, encoded_value(double_min), encoded_value(double_max), 2, 0, 0, - true, true, false); - const auto converted_finite = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - finite_stats); - EXPECT_TRUE(converted_finite.has_min_max); -} - -TEST(ParquetStatisticsTransformTest, IgnoresNaNFloatAndDoubleColumnIndexMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float32(), false), - arrow::field("d", arrow::float64(), false)}), - {float_array({1.0F, 2.0F}), double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - auto float_index = std::make_shared>( - 1.0F, std::numeric_limits::quiet_NaN()); - format::parquet::ParquetColumnStatistics float_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - float_index, *schema[0], 0, &float_page_stats)); - EXPECT_FALSE(float_page_stats.has_min_max); - EXPECT_TRUE(float_page_stats.has_not_null); - - auto double_index = std::make_shared>( - std::numeric_limits::quiet_NaN(), 2.0); - format::parquet::ParquetColumnStatistics double_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - double_index, *schema[1], 0, &double_page_stats)); - EXPECT_FALSE(double_page_stats.has_min_max); - EXPECT_TRUE(double_page_stats.has_not_null); - - auto finite_index = std::make_shared>(1.0, 2.0); - format::parquet::ParquetColumnStatistics finite_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - finite_index, *schema[1], 0, &finite_page_stats)); - EXPECT_TRUE(finite_page_stats.has_min_max); - - auto mixed_index = std::make_shared>( - std::vector {std::numeric_limits::quiet_NaN(), 1.0}, - std::vector {std::numeric_limits::quiet_NaN(), 2.0}); - format::parquet::ParquetColumnStatistics nan_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 0, &nan_page_stats)); - EXPECT_FALSE(nan_page_stats.has_min_max); - - format::parquet::ParquetColumnStatistics following_page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - mixed_index, *schema[1], 1, &following_page_stats)); - EXPECT_TRUE(following_page_stats.has_min_max); - EXPECT_EQ(following_page_stats.min_value.get(), 1.0); - EXPECT_EQ(following_page_stats.max_value.get(), 2.0); -} - -TEST(ParquetStatisticsTransformTest, IgnoresInvertedFooterAndColumnIndexMinMax) { - auto table = arrow::Table::Make( - arrow::schema( - {arrow::field("i", arrow::int32(), false), - arrow::field("s", arrow::utf8(), false), - arrow::field("ts", arrow::timestamp(arrow::TimeUnit::MICRO, "UTC"), false)}), - {int32_array({1, 2}), string_array({"a", "z"}), timestamp_array({1000000, 2000000})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const int32_t inverted_min = 10; - const int32_t inverted_max = 1; - auto int_stats = ::parquet::MakeStatistics<::parquet::Int32Type>( - schema[0]->descriptor, encoded_value(inverted_min), encoded_value(inverted_max), 2, 0, - 0, true, true, false); - const auto converted_int = format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[0], int_stats); - EXPECT_TRUE(converted_int.has_not_null); - EXPECT_FALSE(converted_int.has_min_max); - - const std::string inverted_string_min = "z"; - const std::string inverted_string_max = "a"; - auto string_stats = ::parquet::MakeStatistics<::parquet::ByteArrayType>( - schema[1]->descriptor, inverted_string_min, inverted_string_max, 2, 0, 0, true, true, - false); - const auto converted_string = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[1], - string_stats); - EXPECT_TRUE(converted_string.has_not_null); - EXPECT_FALSE(converted_string.has_min_max); - - auto int_index = - std::make_shared>(inverted_min, inverted_max); - format::parquet::ParquetColumnStatistics page_stats; - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - int_index, *schema[0], 0, &page_stats)); - EXPECT_TRUE(page_stats.has_not_null); - EXPECT_FALSE(page_stats.has_min_max); - - // These endpoints are inverted within one second. Whole-second validation alone would miss - // the corruption, and TIMESTAMPTZ must reject the same raw inversion before its UTC shortcut. - constexpr int64_t timestamp_min = 1500000; - constexpr int64_t timestamp_max = 1000000; - auto timestamp_stats = ::parquet::MakeStatistics<::parquet::Int64Type>( - schema[2]->descriptor, encoded_value(timestamp_min), encoded_value(timestamp_max), 2, 0, - 0, true, true, false); - auto utc = cctz::utc_time_zone(); - const auto converted_timestamp = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp.has_min_max); - - schema[2]->type = std::make_shared(6); - schema[2]->type_descriptor.doris_type = schema[2]->type; - const auto converted_timestamp_tz = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics( - *schema[2], timestamp_stats, &utc); - EXPECT_FALSE(converted_timestamp_tz.has_min_max); -} - -TEST(ParquetStatisticsTransformTest, PreservesNullCountWhenNaNInvalidatesMinMax) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("f", arrow::float64(), false)}), - {double_array({1.0, 2.0})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - const double nan = std::numeric_limits::quiet_NaN(); - const double max_value = 2.0; - auto footer_stats = ::parquet::MakeStatistics<::parquet::DoubleType>( - schema[0]->descriptor, encoded_value(nan), encoded_value(max_value), 2, 0, 0, true, - true, false); - const auto converted_footer = - format::parquet::ParquetStatisticsUtils::TransformColumnStatistics(*schema[0], - footer_stats); - auto footer_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(converted_footer); - ASSERT_NE(footer_zone_map, nullptr); - EXPECT_TRUE(footer_zone_map->pass_all); - EXPECT_FALSE(footer_zone_map->has_null); - EXPECT_TRUE(footer_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*footer_zone_map, schema[0]->type)); - - ZoneMapEvalContext footer_ctx; - footer_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = footer_zone_map}); - Int32ZoneMapExpr is_null_expr(0, Int32ZoneMapExpr::Op::IS_NULL); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(footer_ctx), ZoneMapFilterResult::kNoMatch); - - auto column_index = std::make_shared>(nan, max_value); - format::parquet::ParquetColumnStatistics page_stats; - ASSERT_TRUE(format::parquet::ParquetStatisticsUtils::TransformColumnIndexStatistics( - column_index, *schema[0], 0, &page_stats)); - auto page_zone_map = format::parquet::ParquetStatisticsUtils::MakeZoneMap(page_stats); - ASSERT_NE(page_zone_map, nullptr); - EXPECT_TRUE(page_zone_map->pass_all); - EXPECT_FALSE(page_zone_map->has_null); - EXPECT_TRUE(page_zone_map->has_not_null); - EXPECT_FALSE(expr_zonemap::range_stats_usable_for_zonemap(*page_zone_map, schema[0]->type)); - - ZoneMapEvalContext page_ctx; - page_ctx.slots.emplace(0, ZoneMapEvalContext::SlotZoneMap {.data_type = schema[0]->type, - .zone_map = page_zone_map}); - EXPECT_EQ(is_null_expr.evaluate_zonemap_filter(page_ctx), ZoneMapFilterResult::kNoMatch); -} - -TEST(ParquetStatisticsPruningTest, ExprZonemapPredicatesAndNullPredicatesPruneRowGroups) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({std::nullopt, std::nullopt, 3, 4, 5, 6})}); - auto reader = make_reader(table, 2, false, true); - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct( - std::make_shared(0, Int32ZoneMapExpr::Op::GE, 5)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({2})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 2); - EXPECT_GT(pruning_stats.filtered_bytes, 0); - - selected.clear(); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NOT_NULL)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1, 2})); - - selected.clear(); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_zonemap_conjunct(std::make_shared( - 0, Int32ZoneMapExpr::Op::IS_NULL)), - nullptr, &selected, false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({0})); -} - -TEST(ParquetStatisticsPruningTest, DictionaryPruningHandlesExcludeIncludeAndUnsupportedPaths) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, false); - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"gamma"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 1); - - auto plain_reader = make_reader(table, 2, false, false); - auto plain_schema = build_file_schema(*plain_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *plain_reader->metadata(), plain_reader.get(), plain_schema, - request_with_dictionary_conjunct({"missing"}), nullptr, &selected, false, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 0); -} - -TEST(ParquetStatisticsPruningTest, NativeContradictoryNullMetadataFallsBackForNullPredicates) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("i", arrow::int32(), true)}), - {int32_array({1})}); - auto reader = make_reader(table, 1, false, true); - auto schema = build_file_schema(*reader); - - auto make_indexes = [](int64_t null_count) { - format::parquet::NativeParquetPageIndex indexes; - indexes.column_index.__set_null_pages({true}); - indexes.column_index.__set_null_counts({null_count}); - tparquet::PageLocation location; - location.__set_offset(0); - location.__set_compressed_page_size(1); - location.__set_first_row_index(0); - indexes.offset_index.__set_page_locations({location}); - return std::unordered_map { - {0, std::move(indexes)}}; - }; - - for (const auto op : {Int32ZoneMapExpr::Op::IS_NULL, Int32ZoneMapExpr::Op::IS_NOT_NULL}) { - std::vector selected; - const int64_t null_count = op == Int32ZoneMapExpr::Op::IS_NULL ? 0 : -1; - ASSERT_TRUE( - format::parquet::select_row_group_ranges_by_native_page_index( - make_indexes(null_count), schema, - request_with_zonemap_conjunct(std::make_shared(0, op)), 1, - &selected, nullptr, nullptr) - .ok()); - ASSERT_EQ(selected.size(), 1); - EXPECT_EQ(selected[0].start, 0); - EXPECT_EQ(selected[0].length, 1); - } -} - -TEST(ParquetStatisticsPruningTest, VExprUsesDictionaryAndMissingBloomKeepsRows) { - auto table = arrow::Table::Make(arrow::schema({arrow::field("s", arrow::utf8(), false)}), - {string_array({"alpha", "beta", "gamma", "omega"})}); - auto reader = make_reader(table, 2, true, true); - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *reader->metadata(), reader.get(), schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) - .ok()); - EXPECT_TRUE(selected.empty()); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_statistics, 0); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_dictionary, 2); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - auto no_stats_reader = make_reader(table, 2, false, false); - auto no_stats_schema = build_file_schema(*no_stats_reader); - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( - *no_stats_reader->metadata(), no_stats_reader.get(), no_stats_schema, - request_with_dictionary_conjunct({"absent"}), nullptr, &selected, true, - &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); -} - -TEST(ParquetStatisticsPruningTest, BloomFilterCacheIsScopedToRowGroupAndColumn) { - auto input = arrow::io::ReadableFile::Open( - "./be/test/exec/test_data/parquet_scanner/multi_row_group_bloom_filter.parquet"); - ASSERT_TRUE(input.ok()); - auto reader = ::parquet::ParquetFileReader::Open(*input); - ASSERT_EQ(reader->metadata()->num_row_groups(), 2); - auto& bloom_filter_reader = reader->GetBloomFilterReader(); - for (int row_group_idx = 0; row_group_idx < 2; ++row_group_idx) { - auto row_group_reader = bloom_filter_reader.RowGroup(row_group_idx); - ASSERT_NE(row_group_reader, nullptr); - ASSERT_NE(row_group_reader->GetColumnBloomFilter(0), nullptr); - } - auto schema = build_file_schema(*reader); - - std::vector selected; - format::parquet::ParquetPruningStats pruning_stats; - auto request = request_with_bloom_conjunct(std::make_shared(), - {Field::create_field(12345)}); - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - false, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({0, 1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); - - selected.clear(); - pruning_stats = {}; - ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(*reader->metadata(), reader.get(), - schema, request, nullptr, &selected, - true, &pruning_stats) - .ok()); - EXPECT_EQ(selected, std::vector({1})); - EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 1); -} - -TEST(ParquetBloomFilterPruningTest, VExprEqPrunesAbsentIntValue) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2)}), ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(3)}), ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprInPrunesOnlyWhenAllValuesAreAbsent) { - auto data_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields( - {Field::create_field(1), Field::create_field(3)}, TYPE_INT); - auto ctx = bloom_context(data_type, bloom_filter.get()); - - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(4)}), - ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(data_type, {Field::create_field(2), - Field::create_field(3)}), - ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, VExprBoolAndStringUseSlotBloomFilter) { - auto bool_type = std::make_shared(); - auto bool_filter = - bloom_filter_for_fields({Field::create_field(true)}, TYPE_BOOLEAN); - auto bool_ctx = bloom_context(bool_type, bool_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(false)}), - bool_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(bool_type, {Field::create_field(true)}), - bool_ctx), - ZoneMapFilterResult::kMayMatch); - - auto string_type = std::make_shared(); - auto string_filter = bloom_filter_for_fields( - {Field::create_field("alpha"), Field::create_field("omega")}, - TYPE_STRING); - auto string_ctx = bloom_context(string_type, string_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("beta")}), - string_ctx), - ZoneMapFilterResult::kNoMatch); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(string_type, {Field::create_field("alpha")}), - string_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, MissingOrUnsupportedBloomContextKeepsRowGroup) { - auto int_type = std::make_shared(); - BloomFilterEvalContext missing_ctx; - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(int_type, {Field::create_field(2)}), missing_ctx), - ZoneMapFilterResult::kMayMatch); - - auto smallint_type = std::make_shared(); - auto bloom_filter = bloom_filter_for_fields({Field::create_field(1)}, TYPE_INT); - auto unsupported_ctx = bloom_context(smallint_type, bloom_filter.get()); - EXPECT_EQ(VExprContext::evaluate_bloom_filter( - bloom_conjuncts(smallint_type, {Field::create_field(2)}), - unsupported_ctx), - ZoneMapFilterResult::kMayMatch); -} - -TEST(ParquetBloomFilterPruningTest, ParquetUint32BloomUsesPhysicalInt32Hash) { - const auto column_schema = uint32_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - const uint32_t present_value = 4000000000U; - int32_t physical_value; - memcpy(&physical_value, &present_value, sizeof(physical_value)); - bloom_filter->InsertHash(bloom_filter->Hash(physical_value)); - - // UINT32 is exposed to VExpr as Doris BIGINT, but Parquet stores and hashes it as a physical - // INT32 carrier. A present value above INT32_MAX must therefore be narrowed to the physical - // bit pattern before probing the file bloom filter. - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field( - static_cast(present_value))}), - *bloom_filter)); - - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(-1)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts( - column_schema.type, - {Field::create_field( - static_cast(std::numeric_limits::max()) + 1)}), - *bloom_filter)); -} - TEST(ParquetBloomFilterPruningTest, NativeUint32BloomUsesPhysicalInt32Hash) { const auto column_schema = uint32_parquet_bloom_schema(); format::parquet::native::BlockSplitBloomFilter bloom_filter; @@ -1193,39 +216,9 @@ TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Ma EXPECT_EQ(selected_row_groups, std::vector({0})); EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); } - -TEST(ParquetBloomFilterPruningTest, ParquetFixedLenByteArrayBloomUsesFlbaHash) { - const auto column_schema = fixed_len_string_parquet_bloom_schema(4); - auto bloom_filter = parquet_bloom_filter(); - - const std::string present_value = "abcd"; - ::parquet::FLBA physical_value(reinterpret_cast(present_value.data())); - bloom_filter->InsertHash( - bloom_filter->Hash(&physical_value, column_schema.type_descriptor.fixed_length)); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(present_value)}), - *bloom_filter)); - EXPECT_TRUE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field("abc")}), - *bloom_filter)); -} - -TEST(ParquetBloomFilterPruningTest, ParquetFloat16BloomDoesNotUseFloatHash) { - const auto column_schema = float16_parquet_bloom_schema(); - auto bloom_filter = parquet_bloom_filter(); - - EXPECT_FALSE(format::parquet::ParquetStatisticsUtils::BloomFilterExcludes( - column_schema, 0, - bloom_conjuncts(column_schema.type, {Field::create_field(1.0F)}), - *bloom_filter)); -} - TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { format::parquet::ParquetTypeDescriptor binary_type; - binary_type.physical_type = ::parquet::Type::BYTE_ARRAY; + binary_type.physical_type = tparquet::Type::BYTE_ARRAY; tparquet::Statistics max_only; max_only.__set_max("III"); diff --git a/be/test/format_v2/parquet/parquet_type_test.cpp b/be/test/format_v2/parquet/parquet_type_test.cpp index ecc4fad53ccaf9..0680cc828a9f7a 100644 --- a/be/test/format_v2/parquet/parquet_type_test.cpp +++ b/be/test/format_v2/parquet/parquet_type_test.cpp @@ -17,472 +17,49 @@ #include "format_v2/parquet/parquet_type.h" -#include -#include #include -#include -#include -#include - -#include - -#include "core/data_type/data_type_nullable.h" -#include "core/data_type/primitive_type.h" namespace doris::format::parquet { -namespace { - -::parquet::SchemaDescriptor make_descriptor(const ::parquet::schema::NodePtr& node) { - auto schema = - ::parquet::schema::GroupNode::Make("schema", ::parquet::Repetition::REQUIRED, {node}); - ::parquet::SchemaDescriptor descriptor; - descriptor.Init(schema); - return descriptor; -} - -ParquetTypeDescriptor resolve_node(const ::parquet::schema::NodePtr& node) { - auto descriptor = make_descriptor(node); - return resolve_parquet_type(descriptor.Column(0)); -} - -PrimitiveType primitive_type(const DataTypePtr& type) { - return remove_nullable(type)->get_primitive_type(); -} - -int scale_of(const DataTypePtr& type) { - return remove_nullable(type)->get_scale(); -} - -std::shared_ptr make_float16_array() { - arrow::HalfFloatBuilder builder; - EXPECT_TRUE(builder.Append(0x3E00).ok()); - std::shared_ptr array; - EXPECT_TRUE(builder.Finish(&array).ok()); - return array; -} - -ParquetTypeDescriptor resolve_arrow_float16_type() { - const auto schema = arrow::schema({arrow::field("f16", arrow::float16(), true)}); - const auto table = arrow::Table::Make(schema, {make_float16_array()}); - auto out_result = arrow::io::BufferOutputStream::Create(); - EXPECT_TRUE(out_result.ok()); - auto out = *out_result; - EXPECT_TRUE(::parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), out, 1).ok()); - auto buffer_result = out->Finish(); - EXPECT_TRUE(buffer_result.ok()); - - auto reader = ::parquet::ParquetFileReader::Open( - std::make_shared(*buffer_result)); - return resolve_parquet_type(reader->metadata()->schema()->Column(0)); -} - -} // namespace - -TEST(ParquetTypeTest, ResolveLogicalIntegerMappings) { - struct Case { - int bit_width; - bool is_signed; - PrimitiveType expected_type; - bool expected_unsigned; - }; - const std::vector cases = { - {8, true, TYPE_TINYINT, false}, {8, false, TYPE_SMALLINT, true}, - {16, true, TYPE_SMALLINT, false}, {16, false, TYPE_INT, true}, - {32, true, TYPE_INT, false}, {32, false, TYPE_BIGINT, true}, - {64, true, TYPE_BIGINT, false}, {64, false, TYPE_LARGEINT, true}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.bit_width); - const auto node = ::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Int(test_case.bit_width, test_case.is_signed), - test_case.bit_width == 64 ? ::parquet::Type::INT64 : ::parquet::Type::INT32); - const auto type = resolve_node(node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - } -} - -TEST(ParquetTypeTest, ResolveLogicalTimeAndTimestampMappings) { - const auto time_millis = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - ASSERT_NE(time_millis.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_millis.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_millis.time_unit, ParquetTimeUnit::MILLIS); - EXPECT_EQ(time_millis.extra_type_info, ParquetExtraTypeInfo::UNIT_MS); - - const auto time_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_us", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::MICROS), - ::parquet::Type::INT64)); - ASSERT_NE(time_micros.doris_type, nullptr); - EXPECT_EQ(primitive_type(time_micros.doris_type), TYPE_TIMEV2); - EXPECT_EQ(time_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(time_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); - - const auto adjusted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_adjusted", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::MILLIS), - ::parquet::Type::INT32)); - EXPECT_EQ(adjusted_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_time.unsupported_reason.empty()); - - const auto timestamp_nanos = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ns", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(timestamp_nanos.doris_type, nullptr); - EXPECT_TRUE(timestamp_nanos.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(timestamp_nanos.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(timestamp_nanos.is_timestamp); - EXPECT_TRUE(timestamp_nanos.timestamp_is_adjusted_to_utc); - EXPECT_EQ(timestamp_nanos.time_unit, ParquetTimeUnit::NANOS); - EXPECT_EQ(timestamp_nanos.extra_type_info, ParquetExtraTypeInfo::UNIT_NS); -} - -TEST(ParquetTypeTest, ResolveLogicalTimestampMatrix) { - struct Case { - ::parquet::LogicalType::TimeUnit::unit parquet_unit; - bool adjusted_to_utc; - ParquetTimeUnit expected_unit; - ParquetExtraTypeInfo expected_extra; - int expected_scale; - }; - const std::vector cases = { - {::parquet::LogicalType::TimeUnit::MILLIS, true, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MILLIS, false, ParquetTimeUnit::MILLIS, - ParquetExtraTypeInfo::UNIT_MS, 3}, - {::parquet::LogicalType::TimeUnit::MICROS, true, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::MICROS, false, ParquetTimeUnit::MICROS, - ParquetExtraTypeInfo::UNIT_MICROS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, true, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - {::parquet::LogicalType::TimeUnit::NANOS, false, ParquetTimeUnit::NANOS, - ParquetExtraTypeInfo::UNIT_NS, 6}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_scale); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::OPTIONAL, - ::parquet::LogicalType::Timestamp(test_case.adjusted_to_utc, - test_case.parquet_unit), - ::parquet::Type::INT64)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(type.doris_type), test_case.expected_scale); - EXPECT_TRUE(type.is_timestamp); - EXPECT_EQ(type.timestamp_is_adjusted_to_utc, test_case.adjusted_to_utc); - EXPECT_EQ(type.time_unit, test_case.expected_unit); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ConvertedTimeIsRejectedButConvertedTimestampIsSupported) { - const auto converted_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::TIME_MILLIS)); - EXPECT_EQ(converted_time.doris_type, nullptr); - EXPECT_FALSE(converted_time.unsupported_reason.empty()); - - const auto converted_timestamp = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_ms", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MILLIS)); - ASSERT_NE(converted_timestamp.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_timestamp.doris_type), TYPE_DATETIMEV2); - EXPECT_TRUE(converted_timestamp.is_timestamp); - EXPECT_TRUE(converted_timestamp.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp.time_unit, ParquetTimeUnit::MILLIS); - const auto converted_timestamp_micros = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts_us", ::parquet::Repetition::OPTIONAL, ::parquet::Type::INT64, - ::parquet::ConvertedType::TIMESTAMP_MICROS)); - ASSERT_NE(converted_timestamp_micros.doris_type, nullptr); - EXPECT_TRUE(converted_timestamp_micros.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(converted_timestamp_micros.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(scale_of(converted_timestamp_micros.doris_type), 6); - EXPECT_TRUE(converted_timestamp_micros.is_timestamp); - EXPECT_TRUE(converted_timestamp_micros.timestamp_is_adjusted_to_utc); - EXPECT_EQ(converted_timestamp_micros.time_unit, ParquetTimeUnit::MICROS); - EXPECT_EQ(converted_timestamp_micros.extra_type_info, ParquetExtraTypeInfo::UNIT_MICROS); -} - -TEST(ParquetTypeTest, ResolveConvertedIntegerMappingsAndDecodedKinds) { - struct Case { - ::parquet::ConvertedType::type converted_type; - ::parquet::Type::type physical_type; - PrimitiveType expected_type; - int bit_width; - bool expected_unsigned; - DecodedValueKind expected_value_kind; - }; - const std::vector cases = { - {::parquet::ConvertedType::INT_8, ::parquet::Type::INT32, TYPE_TINYINT, 8, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_8, ::parquet::Type::INT32, TYPE_SMALLINT, 8, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_16, ::parquet::Type::INT32, TYPE_SMALLINT, 16, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_16, ::parquet::Type::INT32, TYPE_INT, 16, true, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::INT_32, ::parquet::Type::INT32, TYPE_INT, 32, false, - DecodedValueKind::INT32}, - {::parquet::ConvertedType::UINT_32, ::parquet::Type::INT32, TYPE_BIGINT, 32, true, - DecodedValueKind::UINT32}, - {::parquet::ConvertedType::INT_64, ::parquet::Type::INT64, TYPE_BIGINT, 64, false, - DecodedValueKind::INT64}, - {::parquet::ConvertedType::UINT_64, ::parquet::Type::INT64, TYPE_LARGEINT, 64, true, - DecodedValueKind::UINT64}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.converted_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, test_case.physical_type, - test_case.converted_type)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(type.integer_bit_width, test_case.bit_width); - EXPECT_EQ(type.is_unsigned_integer, test_case.expected_unsigned); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_value_kind); - } -} - -TEST(ParquetTypeTest, ResolveConvertedDecimalCarriers) { - struct Case { - ::parquet::Type::type physical_type; - int type_length; - int precision; - int scale; - PrimitiveType expected_type; - ParquetExtraTypeInfo expected_extra; - }; - const std::vector cases = { - {::parquet::Type::INT32, -1, 9, 2, TYPE_DECIMAL32, ParquetExtraTypeInfo::DECIMAL_INT32}, - {::parquet::Type::INT64, -1, 18, 6, TYPE_DECIMAL64, - ParquetExtraTypeInfo::DECIMAL_INT64}, - {::parquet::Type::BYTE_ARRAY, -1, 20, 5, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16, 38, 6, TYPE_DECIMAL128I, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - {::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20, 39, 6, TYPE_DECIMAL256, - ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY}, - }; - - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.physical_type); - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, test_case.physical_type, - ::parquet::ConvertedType::DECIMAL, test_case.type_length, test_case.precision, - test_case.scale)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_TRUE(type.is_decimal); - EXPECT_FALSE(type.is_string_like); - EXPECT_EQ(type.decimal_precision, test_case.precision); - EXPECT_EQ(type.decimal_scale, test_case.scale); - EXPECT_EQ(type.extra_type_info, test_case.expected_extra); - } -} - -TEST(ParquetTypeTest, ResolveLogicalStringDateAndDecimalMappings) { - const std::vector> string_like_logical_types = { - ::parquet::LogicalType::String(), ::parquet::LogicalType::Enum(), - ::parquet::LogicalType::JSON(), ::parquet::LogicalType::BSON()}; - for (const auto& logical_type : string_like_logical_types) { - const auto type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::OPTIONAL, logical_type, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_TRUE(type.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(type.doris_type), TYPE_STRING); - EXPECT_TRUE(type.is_string_like); - } - - const auto uuid = resolve_node(::parquet::schema::PrimitiveNode::Make( - "uuid", ::parquet::Repetition::OPTIONAL, ::parquet::LogicalType::UUID(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(uuid.doris_type, nullptr); - EXPECT_TRUE(uuid.doris_type->is_nullable()); - EXPECT_EQ(primitive_type(uuid.doris_type), TYPE_STRING); - EXPECT_TRUE(uuid.is_string_like); - - const auto date = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Date(), - ::parquet::Type::INT32)); - ASSERT_NE(date.doris_type, nullptr); - EXPECT_EQ(primitive_type(date.doris_type), TYPE_DATEV2); - - const auto decimal64 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d64", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(18, 6), - ::parquet::Type::INT64)); - ASSERT_NE(decimal64.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal64.doris_type), TYPE_DECIMAL64); - EXPECT_TRUE(decimal64.is_decimal); - EXPECT_EQ(decimal64.decimal_precision, 18); - EXPECT_EQ(decimal64.decimal_scale, 6); - EXPECT_EQ(decimal64.extra_type_info, ParquetExtraTypeInfo::DECIMAL_INT64); - - const auto decimal128 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d128", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(38, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 16)); - ASSERT_NE(decimal128.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal128.doris_type), TYPE_DECIMAL128I); - EXPECT_TRUE(decimal128.is_decimal); - EXPECT_EQ(decimal128.decimal_precision, 38); - EXPECT_EQ(decimal128.decimal_scale, 6); - EXPECT_EQ(decimal128.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d256", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Decimal(39, 6), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 20)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - EXPECT_FALSE(decimal256.is_string_like); -} - -TEST(ParquetTypeTest, LogicalConvertedAndPhysicalFallbackLevelsAreDistinct) { - const auto logical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::LogicalType::Int(8, true), - ::parquet::Type::INT32)); - ASSERT_NE(logical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(logical_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(logical_type.integer_bit_width, 8); - - const auto converted_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32, - ::parquet::ConvertedType::INT_8)); - ASSERT_NE(converted_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(converted_type.doris_type), TYPE_TINYINT); - EXPECT_EQ(converted_type.integer_bit_width, 8); - - const auto physical_type = resolve_node(::parquet::schema::PrimitiveNode::Make( - "c", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT32)); - ASSERT_NE(physical_type.doris_type, nullptr); - EXPECT_EQ(primitive_type(physical_type.doris_type), TYPE_INT); - EXPECT_EQ(physical_type.integer_bit_width, -1); -} - -TEST(ParquetTypeTest, ResolveDecimalStringLikeFloat16AndPhysicalFallback) { - const auto decimal256 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "d", ::parquet::Repetition::REQUIRED, ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::DECIMAL, 20, 39, 6)); - ASSERT_NE(decimal256.doris_type, nullptr); - EXPECT_EQ(primitive_type(decimal256.doris_type), TYPE_DECIMAL256); - EXPECT_TRUE(decimal256.is_decimal); - EXPECT_FALSE(decimal256.is_string_like); - EXPECT_EQ(decimal256.decimal_precision, 39); - EXPECT_EQ(decimal256.decimal_scale, 6); - EXPECT_EQ(decimal256.extra_type_info, ParquetExtraTypeInfo::DECIMAL_BYTE_ARRAY); - - const auto plain_binary = resolve_node(::parquet::schema::PrimitiveNode::Make( - "s", ::parquet::Repetition::REQUIRED, ::parquet::Type::BYTE_ARRAY)); - ASSERT_NE(plain_binary.doris_type, nullptr); - EXPECT_EQ(primitive_type(plain_binary.doris_type), TYPE_STRING); - EXPECT_TRUE(plain_binary.is_string_like); - - const auto float16 = resolve_arrow_float16_type(); - ASSERT_NE(float16.doris_type, nullptr); - EXPECT_TRUE(float16.doris_type->is_nullable()); - EXPECT_EQ(float16.physical_type, ::parquet::Type::FIXED_LEN_BYTE_ARRAY); - EXPECT_EQ(float16.fixed_length, 2); - EXPECT_EQ(primitive_type(float16.doris_type), TYPE_FLOAT); - EXPECT_EQ(float16.extra_type_info, ParquetExtraTypeInfo::FLOAT16); - EXPECT_FALSE(float16.is_string_like); - EXPECT_EQ(decoded_value_kind(float16), DecodedValueKind::FIXED_BINARY); -} - -TEST(ParquetTypeTest, ResolveNullDescriptorAndPhysicalFallback) { - const auto null_type = resolve_parquet_type(nullptr); - EXPECT_EQ(null_type.doris_type, nullptr); - EXPECT_EQ(null_type.physical_type, ::parquet::Type::UNDEFINED); - - const auto int96 = resolve_node(::parquet::schema::PrimitiveNode::Make( - "ts", ::parquet::Repetition::REQUIRED, ::parquet::Type::INT96)); - ASSERT_NE(int96.doris_type, nullptr); - EXPECT_EQ(primitive_type(int96.doris_type), TYPE_DATETIMEV2); - EXPECT_EQ(int96.extra_type_info, ParquetExtraTypeInfo::IMPALA_TIMESTAMP); - EXPECT_EQ(decoded_value_kind(int96), DecodedValueKind::INT96); -} - -TEST(ParquetTypeTest, ResolveEveryPhysicalFallback) { - struct Case { - ::parquet::schema::NodePtr node; - PrimitiveType expected_type; - DecodedValueKind expected_kind; - bool expected_string_like = false; - }; - const std::vector cases = { - {::parquet::schema::PrimitiveNode::Make("b", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BOOLEAN), - TYPE_BOOLEAN, DecodedValueKind::BOOL}, - {::parquet::schema::PrimitiveNode::Make("i32", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT32), - TYPE_INT, DecodedValueKind::INT32}, - {::parquet::schema::PrimitiveNode::Make("i64", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT64), - TYPE_BIGINT, DecodedValueKind::INT64}, - {::parquet::schema::PrimitiveNode::Make("f", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FLOAT), - TYPE_FLOAT, DecodedValueKind::FLOAT}, - {::parquet::schema::PrimitiveNode::Make("d", ::parquet::Repetition::REQUIRED, - ::parquet::Type::DOUBLE), - TYPE_DOUBLE, DecodedValueKind::DOUBLE}, - {::parquet::schema::PrimitiveNode::Make("s", ::parquet::Repetition::REQUIRED, - ::parquet::Type::BYTE_ARRAY), - TYPE_STRING, DecodedValueKind::BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("fs", ::parquet::Repetition::REQUIRED, - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, - ::parquet::ConvertedType::NONE, 4), - TYPE_STRING, DecodedValueKind::FIXED_BINARY, true}, - {::parquet::schema::PrimitiveNode::Make("ts", ::parquet::Repetition::REQUIRED, - ::parquet::Type::INT96), - TYPE_DATETIMEV2, DecodedValueKind::INT96}, +TEST(ParquetTypeTest, DecodedValueKindUsesNativePhysicalTypes) { + ParquetTypeDescriptor descriptor; + + const std::pair cases[] = { + {tparquet::Type::BOOLEAN, DecodedValueKind::BOOL}, + {tparquet::Type::INT32, DecodedValueKind::INT32}, + {tparquet::Type::INT64, DecodedValueKind::INT64}, + {tparquet::Type::INT96, DecodedValueKind::INT96}, + {tparquet::Type::FLOAT, DecodedValueKind::FLOAT}, + {tparquet::Type::DOUBLE, DecodedValueKind::DOUBLE}, + {tparquet::Type::BYTE_ARRAY, DecodedValueKind::BINARY}, + {tparquet::Type::FIXED_LEN_BYTE_ARRAY, DecodedValueKind::FIXED_BINARY}, }; - for (const auto& test_case : cases) { - SCOPED_TRACE(test_case.expected_type); - const auto type = resolve_node(test_case.node); - ASSERT_NE(type.doris_type, nullptr); - EXPECT_EQ(primitive_type(type.doris_type), test_case.expected_type); - EXPECT_EQ(decoded_value_kind(type), test_case.expected_kind); - EXPECT_EQ(type.is_string_like, test_case.expected_string_like); + for (const auto& [physical_type, expected] : cases) { + descriptor.physical_type = physical_type; + descriptor.is_unsigned_integer = false; + descriptor.integer_bit_width = -1; + EXPECT_EQ(decoded_value_kind(descriptor), expected); } } -TEST(ParquetTypeTest, InvalidLogicalAnnotationsFallBackOrRejectAsSpecified) { - EXPECT_THROW(::parquet::LogicalType::Int(24, true), ::parquet::ParquetException); +TEST(ParquetTypeTest, DecodedValueKindPreservesUnsignedWidth) { + ParquetTypeDescriptor descriptor; + descriptor.is_unsigned_integer = true; - const auto nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(false, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - ASSERT_NE(nanos_time.doris_type, nullptr); - EXPECT_EQ(primitive_type(nanos_time.doris_type), TYPE_BIGINT); - EXPECT_TRUE(nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 32; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT32); - const auto adjusted_nanos_time = resolve_node(::parquet::schema::PrimitiveNode::Make( - "time_ns_utc", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Time(true, ::parquet::LogicalType::TimeUnit::NANOS), - ::parquet::Type::INT64)); - EXPECT_EQ(adjusted_nanos_time.doris_type, nullptr); - EXPECT_FALSE(adjusted_nanos_time.unsupported_reason.empty()); + descriptor.physical_type = tparquet::Type::INT64; + descriptor.integer_bit_width = 64; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::UINT64); - EXPECT_THROW(::parquet::schema::PrimitiveNode::Make("f16_bad", ::parquet::Repetition::REQUIRED, - ::parquet::LogicalType::Float16(), - ::parquet::Type::FIXED_LEN_BYTE_ARRAY, 4), - ::parquet::ParquetException); + // Narrow unsigned logical integers still use the signed physical decoder; conversion + // happens after decoding so the on-disk bit width remains the single source of truth. + descriptor.physical_type = tparquet::Type::INT32; + descriptor.integer_bit_width = 16; + EXPECT_EQ(decoded_value_kind(descriptor), DecodedValueKind::INT32); } } // namespace doris::format::parquet From 28878768a56b481d7873f5c1f76d7543e1907f7d Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 18:02:16 +0800 Subject: [PATCH 22/34] [chore](be) clarify native Parquet V2 I/O comments --- .../format_v2/parquet/parquet_file_context.h | 20 +++++++++---------- be/src/format_v2/parquet/parquet_profile.h | 7 +++---- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/be/src/format_v2/parquet/parquet_file_context.h b/be/src/format_v2/parquet/parquet_file_context.h index fdaaeaa3c2a5fa..0cd413e10557ac 100644 --- a/be/src/format_v2/parquet/parquet_file_context.h +++ b/be/src/format_v2/parquet/parquet_file_context.h @@ -101,14 +101,14 @@ bool is_serialized_index_span_safe(int64_t span_offset, int64_t span_end); std::vector valid_prefetch_ranges( const std::vector& ranges); -// Average projected column-chunk size for one row group. The v1 parquet path uses this value to -// decide whether a row group is dominated by small random IOs; v2 uses the same signal before -// installing MergeRangeFileReader. Example: chunks of 512KB and 1MB average below SMALL_IO and are -// good merge-reader candidates, while two 8MB chunks should stay on the raw random-access reader. +// Average projected column-chunk size for one row group. V2 uses this signal to decide whether the +// row group is dominated by small random IOs before installing MergeRangeFileReader. Example: +// chunks of 512KB and 1MB average below SMALL_IO and are good merge-reader candidates, while two +// 8MB chunks should stay on the raw random-access reader. size_t average_prefetch_range_size(const std::vector& ranges); -// Decide whether Arrow ReadAt() should be routed through MergeRangeFileReader for the current row -// group. This is intentionally stricter than the background warm-up path: +// Decide whether native data-page ReadAt() should be routed through MergeRangeFileReader for the +// current row group. This is intentionally stricter than the background warm-up path: // - no valid projected chunks -> nothing to merge; // - in-memory file readers already avoid remote random IO; // - average chunk size >= MergeRangeFileReader::SMALL_IO would make merged reading wasteful. @@ -126,8 +126,8 @@ struct ParquetFileContext { // Native metadata, index, and data-page paths share Doris' FileReader without transferring // ownership to an external metadata tree. io::FileReaderSPtr native_file; - // Row-group-scoped view of native_file. Small projected chunks use the same - // MergeRangeFileReader policy as v1; large chunks and in-memory files keep native_file. + // Row-group-scoped view of native_file. Small projected chunks use MergeRangeFileReader; + // large chunks and in-memory files keep native_file. io::FileReaderSPtr native_row_group_file; io::IOContext* native_io_ctx = nullptr; // V2-owned Thrift footer/schema used to construct native page/encoding readers. A cache hit is @@ -156,8 +156,8 @@ struct ParquetFileContext { // random-access behavior and simply skip prefetch. void prefetch_ranges(const std::vector& ranges, const io::IOContext* io_ctx); - // Install the v1-compatible MergeRangeFileReader on the native data-page path. Dictionary - // probes must run before this method because their Arrow ReadAt order is independent of the + // Install the row-group-scoped MergeRangeFileReader on the native data-page path. Dictionary + // probes must run before this method because their native ReadAt order is independent of the // sequential projected chunk ranges consumed by MergeRangeFileReader. bool set_native_random_access_ranges(const std::vector& ranges, size_t avg_io_size, RuntimeProfile* profile, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index d17939e70e7a41..afd77c8996e614 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -41,9 +41,8 @@ struct ParquetColumnReaderProfile { RuntimeProfile::Counter* hybrid_selection_batches = nullptr; RuntimeProfile::Counter* hybrid_selection_ranges = nullptr; RuntimeProfile::Counter* hybrid_selection_null_fallback_batches = nullptr; - // Native page/encoding reader internals. These counters intentionally mirror v1 so a v1/v2 - // profile comparison attributes page IO, decompression, levels, value decode and conversion to - // the same stages. + // Native page/encoding reader internals. These counters keep page IO, decompression, levels, + // value decode and conversion attributable to separate stages. RuntimeProfile::Counter* decompress_time = nullptr; RuntimeProfile::Counter* decompress_count = nullptr; RuntimeProfile::Counter* decode_header_time = nullptr; @@ -64,7 +63,7 @@ struct ParquetColumnReaderProfile { RuntimeProfile::Counter* page_cache_miss_count = nullptr; RuntimeProfile::Counter* page_cache_compressed_hit_count = nullptr; RuntimeProfile::Counter* page_cache_decompressed_hit_count = nullptr; - RuntimeProfile::Counter* native_read_calls = nullptr; // v1-native reader calls + RuntimeProfile::Counter* native_read_calls = nullptr; // native column-reader calls RuntimeProfile::Counter* native_page_fragments = nullptr; // page-bounded read fragments RuntimeProfile::Counter* page_crossing_batches = nullptr; // batches spanning multiple pages RuntimeProfile::Counter* nested_batches = nullptr; // complex-column read batches From 7b749263c4035ae88bedf4fa80663e743a8dba13 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 19:03:17 +0800 Subject: [PATCH 23/34] [fix](be) preserve active Parquet delta decoder state Defer reclaiming DELTA_BYTE_ARRAY prefix state and DELTA_BINARY_PACKED miniblock tables until the current page is exhausted. Add ASAN-covered multi-batch regressions for both decoder reuse paths. --- .../reader/native/delta_bit_pack_decoder.h | 16 +++-- .../format_v2/parquet/native_decoder_test.cpp | 69 +++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h index b3abc55b18d03f..26e13119b8cdfb 100644 --- a/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h +++ b/be/src/format_v2/parquet/reader/native/delta_bit_pack_decoder.h @@ -142,7 +142,11 @@ class DeltaBitPackDecoder final : public DeltaDecoder { void release_scratch(size_t max_retained_bytes) override { release_vector_if_oversized(&_values, max_retained_bytes); - release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + // The bit-width table drives later miniblock transitions, so it is reclaimable only after + // the page is exhausted; _values contains completed batch output and is always disposable. + if (_total_values_remaining == 0) { + release_vector_if_oversized(&_delta_bit_widths, max_retained_bytes); + } } size_t retained_scratch_bytes() const override { return _values.capacity() * sizeof(T) + _delta_bit_widths.capacity() * sizeof(uint8_t); @@ -462,9 +466,13 @@ class DeltaByteArrayDecoder : public DeltaDecoder { release_vector_if_oversized(&_buffered_data, max_retained_bytes); _prefix_len_decoder.release_scratch(max_retained_bytes); _suffix_decoder.release_scratch(max_retained_bytes); - if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); - if (_last_value_in_previous_page.capacity() > max_retained_bytes) { - std::string().swap(_last_value_in_previous_page); + // Prefix reconstruction crosses batch boundaries, so the previous value remains semantic + // decoder state until every value in the current page has been consumed. + if (_num_valid_values == 0) { + if (_last_value.capacity() > max_retained_bytes) std::string().swap(_last_value); + if (_last_value_in_previous_page.capacity() > max_retained_bytes) { + std::string().swap(_last_value_in_previous_page); + } } } size_t retained_scratch_bytes() const override { diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 1c8687a4606fa6..d7283309a90f1c 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -1526,6 +1526,75 @@ TEST(ParquetV2NativeDecoderTest, DecoderOwnedHighWaterScratchIsReleased) { EXPECT_LE(decoder->retained_scratch_bytes(), 64UL << 10); } +TEST(ParquetV2NativeDecoderTest, DeltaByteArrayScratchReleasePreservesPrefixState) { + const std::vector values {std::string(4096, 'x'), "shared-prefix-a", + "shared-prefix-b", "shared-prefix-c"}; + std::vector<::parquet::ByteArray> byte_arrays; + for (const auto& value : values) { + byte_arrays.emplace_back(static_cast(value.size()), + reinterpret_cast(value.data())); + } + auto byte_descriptor = descriptor(::parquet::Type::BYTE_ARRAY); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::ByteArrayType>( + ::parquet::Encoding::DELTA_BYTE_ARRAY, false, byte_descriptor.get()); + encoder->Put(byte_arrays.data(), static_cast(byte_arrays.size())); + auto encoded = encoder->FlushValues(); + + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::BYTE_ARRAY, + tparquet::Encoding::DELTA_BYTE_ARRAY, decoder) + .ok()); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + CaptureBinaryConsumer consumer; + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + ASSERT_TRUE(decoder->decode_binary_values(1, consumer).ok()); + decoder->release_scratch(64); + ASSERT_TRUE(decoder->decode_binary_values(2, consumer).ok()); + ASSERT_EQ(consumer.refs.size(), 2); + EXPECT_EQ(consumer.refs[0].to_string_view(), values[2]); + EXPECT_EQ(consumer.refs[1].to_string_view(), values[3]); +} + +TEST(ParquetV2NativeDecoderTest, DeltaBitPackScratchReleasePreservesMiniblockState) { + std::unique_ptr decoder; + ASSERT_TRUE(Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED, + decoder) + .ok()); + + std::vector outlier_header(64); + uint8_t* cursor = outlier_header.data(); + cursor = encode_varint32(cursor, 512); // values per block + cursor = encode_varint32(cursor, 16); // miniblocks per block + cursor = encode_varint32(cursor, 2); // total values + cursor = encode_varint32(cursor, 0); // first value, zig-zag encoded + cursor = encode_varint32(cursor, 0); // minimum delta, zig-zag encoded + memset(cursor, 0, 16); // one zero bit width per miniblock + cursor += 16; + outlier_header.resize(cursor - outlier_header.data()); + decoder->set_expected_values(2); + Slice outlier_slice(outlier_header.data(), outlier_header.size()); + ASSERT_TRUE(decoder->set_data(&outlier_slice).ok()); + CaptureFixedConsumer consumer; + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + + std::vector values(40); + std::iota(values.begin(), values.end(), 0); + auto int_descriptor = descriptor(::parquet::Type::INT32); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, int_descriptor.get()); + encoder->Put(values.data(), static_cast(values.size())); + auto encoded = encoder->FlushValues(); + decoder->set_expected_values(values.size()); + Slice slice(encoded->data(), encoded->size()); + ASSERT_TRUE(decoder->set_data(&slice).ok()); + ASSERT_TRUE(decoder->decode_fixed_values(1, consumer).ok()); + decoder->release_scratch(8); + ASSERT_TRUE(decoder->decode_fixed_values(values.size() - 1, consumer).ok()); + EXPECT_EQ(consumer.values().size(), values.size() + 1); +} + TEST(ParquetV2NativeDecoderTest, PageHeaderRejectsSignedAndV2LevelSizeCorruption) { auto parse_header = [](tparquet::PageHeader header) { std::vector bytes; From 6bc80731bffdb12b4dd3d135e2ed8713b2e8892c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sun, 19 Jul 2026 21:04:44 +0800 Subject: [PATCH 24/34] [fix](be) address native scan review findings --- .../data_type_datetimev2_serde.cpp | 13 +--- .../data_type_datev2_serde.cpp | 13 +--- .../data_type_decimal_serde.cpp | 13 +--- .../data_type_number_serde.cpp | 14 +--- .../data_type_serde/data_type_time_serde.cpp | 13 +--- .../data_type_timestamptz_serde.cpp | 13 +--- .../data_type_serde/parquet_decode_source.h | 42 ++++++++-- be/src/format_v2/orc/orc_reader.cpp | 22 +++--- be/src/format_v2/parquet/parquet_reader.cpp | 3 +- be/src/format_v2/parquet/parquet_scan.cpp | 7 +- be/src/format_v2/parquet/parquet_scan.h | 3 +- .../format_v2/parquet/parquet_statistics.cpp | 14 ++-- be/src/format_v2/parquet/parquet_statistics.h | 4 +- .../reader/native/column_chunk_reader.cpp | 28 +++---- be/test/format_v2/orc/orc_reader_test.cpp | 77 +++++++++++++++---- .../format_v2/parquet/native_decoder_test.cpp | 51 ++++++++++++ .../format_v2/parquet/parquet_reader_test.cpp | 60 +++++++++++++-- 17 files changed, 253 insertions(+), 137 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index 677bdc0a080afb..fcf369d4263f38 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -707,18 +707,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure()) { - for (size_t row = 0; row < num_values; ++row) { - if (!state.dictionary_conversion_failures.empty() && - state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - state.mark_conversion_failure(old_size + row); - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp index c7e0478c156053..c54bd1dd678823 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp @@ -230,18 +230,7 @@ Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure()) { - for (size_t row = 0; row < num_values; ++row) { - if (!state.dictionary_conversion_failures.empty() && - state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - state.mark_conversion_failure(old_size + row); - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index 5648406a974628..6174507de54360 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -789,18 +789,7 @@ Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure()) { - for (size_t row = 0; row < num_values; ++row) { - if (!state.dictionary_conversion_failures.empty() && - state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - state.mark_conversion_failure(old_size + row); - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } template diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 6ef52bedd42b10..20d7771209a4e5 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -619,19 +619,7 @@ Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure() && - !state.dictionary_conversion_failures.empty()) { - for (size_t row = 0; row < num_values; ++row) { - if (state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - DORIS_CHECK_LT(old_size + row, state.conversion_failure_null_map->size()); - (*state.conversion_failure_null_map)[old_size + row] = 1; - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } } diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index 7fb505085c88c6..062edd756ceacb 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -289,18 +289,7 @@ Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure()) { - for (size_t row = 0; row < num_values; ++row) { - if (!state.dictionary_conversion_failures.empty() && - state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - state.mark_conversion_failure(old_size + row); - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } template diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index 4278a8f30f55ab..aab8adc4e98988 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -436,18 +436,7 @@ Status DataTypeTimeStampTzSerDe::read_column_from_parquet( } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure()) { - for (size_t row = 0; row < num_values; ++row) { - if (!state.dictionary_conversion_failures.empty() && - state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - state.mark_conversion_failure(old_size + row); - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index b9a3a8c731bb6e..1c56f4b10e4769 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -232,16 +232,19 @@ struct ParquetMaterializationState { bool enable_strict_mode = false; IColumn::Filter* conversion_failure_null_map = nullptr; IColumn::Filter dictionary_conversion_failures; + bool capturing_dictionary_conversion_failures = false; void reset_dictionary() { typed_dictionary.reset(); dictionary_indices.clear(); dictionary_conversion_failures.clear(); + capturing_dictionary_conversion_failures = false; dictionary_generation = std::numeric_limits::max(); } bool can_insert_null_on_conversion_failure() const { - return !enable_strict_mode && conversion_failure_null_map != nullptr; + return conversion_failure_null_map != nullptr && + (!enable_strict_mode || capturing_dictionary_conversion_failures); } bool mark_conversion_failure(size_t output_row) { @@ -255,18 +258,41 @@ struct ParquetMaterializationState { IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { auto* output_null_map = conversion_failure_null_map; - dictionary_conversion_failures.clear(); - if (can_insert_null_on_conversion_failure()) { - // Only nullable non-strict outputs may absorb a bad dictionary entry. Redirecting a - // non-nullable decode here would silently turn a required error into a default value. - dictionary_conversion_failures.resize_fill(dictionary_size, 0); - conversion_failure_null_map = &dictionary_conversion_failures; - } + dictionary_conversion_failures.resize_fill(dictionary_size, 0); + conversion_failure_null_map = &dictionary_conversion_failures; + capturing_dictionary_conversion_failures = true; return output_null_map; } void end_dictionary_conversion(IColumn::Filter* output_null_map) { conversion_failure_null_map = output_null_map; + capturing_dictionary_conversion_failures = false; + } + + Status materialize_dictionary(IColumn& column) { + const size_t old_size = column.size(); + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] != 0 && + !can_insert_null_on_conversion_failure()) { + // A malformed dictionary entry is irrelevant until a selected row references it; + // failing while building the dictionary would reject otherwise valid pages. + return Status::DataQualityError( + "Parquet dictionary entry {} cannot be converted to the target type", + dictionary_id); + } + } + column.insert_indices_from(*typed_dictionary, dictionary_indices.data(), + dictionary_indices.data() + dictionary_indices.size()); + if (can_insert_null_on_conversion_failure()) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + if (dictionary_conversion_failures[dictionary_indices[row]] != 0) { + mark_conversion_failure(old_size + row); + } + } + } + return Status::OK(); } }; diff --git a/be/src/format_v2/orc/orc_reader.cpp b/be/src/format_v2/orc/orc_reader.cpp index 076b79a6885d00..49d878257aa0df 100644 --- a/be/src/format_v2/orc/orc_reader.cpp +++ b/be/src/format_v2/orc/orc_reader.cpp @@ -806,20 +806,22 @@ void OrcReader::_init_profile() { _profile, "EvaluatedRowGroupCount", TUnit::UNIT, orc_profile, 1); _orc_profile.read_row_count = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ReadRowCount", TUnit::UNIT, orc_profile, 1); + // RuntimeProfile counter names are flat; format-qualified names keep ORC ownership stable when + // one scan profile also initializes Parquet counters in either order. _orc_profile.filtered_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFiltered", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _profile, "OrcRowGroupsFiltered", TUnit::UNIT, orc_profile, 1); _orc_profile.filtered_row_groups_by_min_max = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsFilteredByMinMax", TUnit::UNIT, file_scan_profile::FILE_READER, 1); - _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowGroupsReadNum", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _profile, "OrcRowGroupsFilteredByMinMax", TUnit::UNIT, orc_profile, 1); + _orc_profile.read_row_groups = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcRowGroupsReadNum", + TUnit::UNIT, orc_profile, 1); _orc_profile.filtered_group_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByGroup", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _profile, "OrcFilteredRowsByGroup", TUnit::UNIT, orc_profile, 1); _orc_profile.lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredRowsByLazyRead", TUnit::UNIT, file_scan_profile::FILE_READER, 1); - _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "FilteredBytes", TUnit::BYTES, file_scan_profile::FILE_READER, 1); - _orc_profile.open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "FileNum", TUnit::UNIT, - file_scan_profile::FILE_READER, 1); + _profile, "OrcFilteredRowsByLazyRead", TUnit::UNIT, orc_profile, 1); + _orc_profile.filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFilteredBytes", + TUnit::BYTES, orc_profile, 1); + _orc_profile.open_file_num = + ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcFileNum", TUnit::UNIT, orc_profile, 1); } void OrcReader::_collect_profile() const { diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index ed542f5f446e14..ac3db1fd81b83b 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -398,7 +398,8 @@ Status ParquetReader::open(std::shared_ptr request) { RETURN_IF_ERROR(plan_parquet_row_groups( *_state->file_context.native_metadata, _state->file_schema, *request_snapshot, scan_range, _state->enable_bloom_filter, &row_group_plan, _state->timezone, - _state->runtime_state, &_state->file_context)); + _state->runtime_state, &_state->file_context, + _parquet_profile.column_reader_profile())); if (_profile != nullptr) { _parquet_profile.update_pruning_stats(row_group_plan.pruning_stats); } diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 5599b93aa9a623..5326a22de7fc67 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -440,8 +440,8 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const format::FileScanRequest& request, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone, - const RuntimeState* runtime_state, - ParquetFileContext* file_context) { + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile) { DORIS_CHECK(plan != nullptr && file_context != nullptr); plan->row_groups.clear(); plan->pruning_stats = {}; @@ -452,7 +452,8 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, std::vector metadata_selected; RETURN_IF_ERROR(select_row_groups_by_metadata( metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, - enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context)); + enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context, + column_reader_profile)); RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, metadata_selected, row_group_first_rows, plan, timezone, runtime_state, file_context)); diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index e1dfcd5b2258d5..b885dc6ec329d7 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -123,7 +123,8 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, const ParquetScanRange& scan_range, bool enable_bloom_filter, RowGroupScanPlan* plan, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr, - ParquetFileContext* file_context = nullptr); + ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}); IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index 04110306eb6a8f..2bf92987e63e59 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -695,7 +695,7 @@ ParquetRowGroupPruneReason native_dictionary_prune_reason( const tparquet::RowGroup& row_group, int row_group_idx, const std::vector>& file_schema, const format::FileScanRequest& request, const cctz::time_zone* timezone, - ParquetFileContext* file_context) { + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile) { if (file_context == nullptr || file_context->native_metadata == nullptr) { return ParquetRowGroupPruneReason::NONE; } @@ -728,11 +728,13 @@ ParquetRowGroupPruneReason native_dictionary_prune_reason( std::unique_ptr reader; const std::vector ranges {{0, row_group.num_rows}}; const std::unordered_map offset_indexes; + // Metadata pruning uses the real native reader, so its page work must be attributed to the + // scan profile even when the row group is eliminated before execution readers are built. const auto status = NativeColumnReader::create( *column_schema, projection, file_context->native_file, file_context->native_metadata, row_group_idx, ranges, offset_indexes, timezone, file_context->native_io_ctx, nullptr, file_context->native_page_cache_enabled, - file_context->native_page_cache_file_key, true, {}, &reader); + file_context->native_page_cache_file_key, true, column_reader_profile, &reader); if (!status.ok() || reader == nullptr) { continue; } @@ -844,7 +846,8 @@ Status select_row_groups_by_metadata( const format::FileScanRequest& request, const std::vector* candidate_row_groups, std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, - const RuntimeState* runtime_state, ParquetFileContext* file_context) { + const RuntimeState* runtime_state, ParquetFileContext* file_context, + const ParquetColumnReaderProfile& column_reader_profile) { int64_t timer_sink = 0; SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink : &pruning_stats->row_group_filter_time); @@ -871,8 +874,9 @@ Status select_row_groups_by_metadata( prune_reason = ParquetRowGroupPruneReason::STATISTICS; } if (prune_reason == ParquetRowGroupPruneReason::NONE) { - prune_reason = native_dictionary_prune_reason(row_group, row_group_idx, file_schema, - request, timezone, file_context); + prune_reason = + native_dictionary_prune_reason(row_group, row_group_idx, file_schema, request, + timezone, file_context, column_reader_profile); } if (prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 6b13444b0aa818..47eedc39ce3c52 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -30,6 +30,7 @@ #include "core/string_ref.h" #include "exprs/vexpr_fwd.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "format_v2/parquet/selection_vector.h" namespace cctz { @@ -128,7 +129,8 @@ Status select_row_groups_by_metadata( const format::FileScanRequest& request, const std::vector* candidate_row_groups, std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, - const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr); + const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr, + const ParquetColumnReaderProfile& column_reader_profile = {}); Status select_row_group_ranges_by_native_page_index( const std::unordered_map& page_indexes, diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 99b80d81cb8aa1..305fe8db2525bd 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -256,18 +256,7 @@ Status read_v2_int96_datetime(IColumn& column, ParquetDecodeSource& source, } RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - const size_t old_size = column.size(); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - if (state.can_insert_null_on_conversion_failure()) { - for (size_t row = 0; row < num_values; ++row) { - if (!state.dictionary_conversion_failures.empty() && - state.dictionary_conversion_failures[state.dictionary_indices[row]] != 0) { - state.mark_conversion_failure(old_size + row); - } - } - } - return Status::OK(); + return state.materialize_dictionary(column); } Status read_native_or_serde(IColumn& column, const DataTypeSerDe& serde, @@ -894,7 +883,11 @@ Status ColumnChunkReader::_decode_dict_page() { CHECK(_block_compress_codec); // Decompress cached compressed dictionary data Slice dict_slice(dict_data.get(), uncompressed_size); - RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR(_block_compress_codec->decompress(payload_slice, &dict_slice)); + } if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { return Status::Corruption( "Parquet dictionary decompressed to {} bytes, expected {}", @@ -924,7 +917,14 @@ Status ColumnChunkReader::_decode_dict_page() { Slice dict_slice(dict_data.get(), uncompressed_size); if (dict_num != 0) { RETURN_IF_ERROR(_page_reader->get_page_data(compressed_data)); - RETURN_IF_ERROR(_block_compress_codec->decompress(compressed_data, &dict_slice)); + // Dictionary probes stop before data pages, so count their decompression here or + // metadata pruning profiles will report no codec work for the scan. + { + SCOPED_RAW_TIMER(&_chunk_statistics.decompress_time); + ++_chunk_statistics.decompress_cnt; + RETURN_IF_ERROR( + _block_compress_codec->decompress(compressed_data, &dict_slice)); + } if (UNLIKELY(dict_slice.size != static_cast(uncompressed_size))) { return Status::Corruption( "Parquet dictionary decompressed to {} bytes, expected {}", diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index 6809186f115b59..f2a64e26b92d15 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -73,6 +73,7 @@ #include "format_v2/expr/cast.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" +#include "format_v2/parquet/parquet_profile.h" #include "gen_cpp/Types_types.h" #include "io/fs/buffered_reader.h" #include "io/io_common.h" @@ -6701,13 +6702,13 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { ASSERT_EQ(result_rows, 200); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsReadNum"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByGroup"), nullptr); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - ASSERT_NE(profile.get_counter("FilteredBytes"), nullptr); - ASSERT_NE(profile.get_counter("FileNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsReadNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByGroup"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + ASSERT_NE(profile.get_counter("OrcFilteredBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcFileNum"), nullptr); const std::array orc_reader_metric_counters { "ReaderCall", "ReaderInclusiveLatencyUs", @@ -6726,19 +6727,63 @@ TEST_F(NewOrcReaderTest, ClosePublishesReaderStatisticsToRuntimeProfile) { for (const auto counter_name : orc_reader_metric_counters) { ASSERT_NE(profile.get_counter(std::string(counter_name)), nullptr) << counter_name; } - EXPECT_EQ(profile.get_counter("RowGroupsFiltered")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 2); - EXPECT_EQ(profile.get_counter("RowGroupsReadNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFiltered")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcRowGroupsReadNum")->value(), 1); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 1); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 3); - EXPECT_EQ(profile.get_counter("FilteredRowsByGroup")->value(), 400); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); - EXPECT_GT(profile.get_counter("FilteredBytes")->value(), 0); - EXPECT_EQ(profile.get_counter("FileNum")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByGroup")->value(), 400); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); + EXPECT_GT(profile.get_counter("OrcFilteredBytes")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcFileNum")->value(), 1); EXPECT_EQ(profile.get_counter("ReadRowCount")->value(), static_cast(file_reader_stats.read_rows)); } +TEST_F(NewOrcReaderTest, ProfileKeepsPruningCountersBelowOrcReader) { + RuntimeProfile profile("new_orc_reader_hierarchy_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + TRuntimeProfileTree tree; + profile.to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("OrcReader")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsFiltered")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcRowGroupsReadNum")); + EXPECT_TRUE(children.at("OrcReader").contains("OrcFilteredRowsByGroup")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenParquetInitializesFirst) { + RuntimeProfile profile("parquet_then_orc_profile"); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + +TEST_F(NewOrcReaderTest, ProfileCountersStayIsolatedWhenOrcInitializesFirst) { + RuntimeProfile profile("orc_then_parquet_profile"); + auto reader = create_reader(&profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + format::parquet::ParquetProfile parquet_profile; + parquet_profile.init(&profile); + + ASSERT_NE(profile.get_counter("OrcRowGroupsFiltered"), nullptr); + ASSERT_NE(profile.get_counter("RowGroupsFiltered"), nullptr); + EXPECT_NE(profile.get_counter("OrcRowGroupsFiltered"), + profile.get_counter("RowGroupsFiltered")); +} + TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { const auto multi_stripe_file_path = (_test_dir / "profile_minmax_disabled.orc").string(); write_multi_stripe_orc_int_file(multi_stripe_file_path, {1, 1000, 2000}); @@ -6775,10 +6820,10 @@ TEST_F(NewOrcReaderTest, DisableOrcFilterByMinMaxKeepsRowGroupProfileZero) { ASSERT_NE(profile.get_counter("SelectedRowGroupCount"), nullptr); ASSERT_NE(profile.get_counter("EvaluatedRowGroupCount"), nullptr); - ASSERT_NE(profile.get_counter("RowGroupsFilteredByMinMax"), nullptr); + ASSERT_NE(profile.get_counter("OrcRowGroupsFilteredByMinMax"), nullptr); EXPECT_EQ(profile.get_counter("SelectedRowGroupCount")->value(), 0); EXPECT_EQ(profile.get_counter("EvaluatedRowGroupCount")->value(), 0); - EXPECT_EQ(profile.get_counter("RowGroupsFilteredByMinMax")->value(), 0); + EXPECT_EQ(profile.get_counter("OrcRowGroupsFilteredByMinMax")->value(), 0); } TEST_F(NewOrcReaderTest, SargConjunctPrunesStripes) { diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index d7283309a90f1c..56785901df15a0 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -2111,6 +2112,56 @@ TEST(ParquetV2NativeDecoderTest, NullableNumericOverflowIsNullOnlyOutsideStrictM EXPECT_EQ(null_map[0], 0); } +TEST(ParquetV2NativeDecoderTest, DictionaryConversionFailureIsDeferredUntilReferenced) { + auto decode_dictionary_id = [](uint8_t dictionary_id, bool strict, IColumn::Filter* null_map, + MutableColumnPtr* column) { + const std::array dictionary_values {1, 1000}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + RETURN_IF_ERROR(Decoder::get_decoder(tparquet::Type::INT32, + tparquet::Encoding::RLE_DICTIONARY, decoder)); + decoder->set_type_length(sizeof(int32_t)); + RETURN_IF_ERROR( + decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size())); + char encoded_id[] = {1, 2, static_cast(dictionary_id)}; + Slice id_slice(encoded_id, sizeof(encoded_id)); + RETURN_IF_ERROR(decoder->set_data(&id_slice)); + + ParquetDecodeContext context; + context.physical_type = ParquetPhysicalType::INT32; + context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState state; + state.enable_strict_mode = strict; + state.conversion_failure_null_map = null_map; + DataTypeInt8 type; + *column = type.create_column(); + return type.get_serde()->read_column_from_parquet(**column, *decoder, context, 1, state); + }; + + for (const bool strict : {false, true}) { + IColumn::Filter null_map; + IColumn::Filter* output_null_map = nullptr; + if (strict) { + null_map.resize_fill(1, 0); + output_null_map = &null_map; + } + + MutableColumnPtr column; + const auto unused_bad = decode_dictionary_id(0, strict, output_null_map, &column); + ASSERT_TRUE(unused_bad.ok()) << unused_bad; + ASSERT_EQ(column->size(), 1); + EXPECT_EQ(assert_cast(*column).get_element(0), 1); + + const auto referenced_bad = decode_dictionary_id(1, strict, output_null_map, &column); + EXPECT_TRUE(referenced_bad.is()) << referenced_bad; + EXPECT_TRUE(column->empty()); + if (output_null_map != nullptr) { + EXPECT_EQ((*output_null_map)[0], 0); + } + } +} + TEST(ParquetV2NativeDecoderTest, FixedLengthStringsAppendAsOneContiguousSpan) { ColumnString column; const std::string values = "aaabbbccc"; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 10bdc2d8a935dd..03f29b2a8856b8 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -35,6 +35,7 @@ #include #include +#include "common/config.h" #include "core/assert_cast.h" #include "core/block/block.h" #include "core/column/column_array.h" @@ -71,6 +72,7 @@ #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/segment/condition_cache.h" #include "storage/utils.h" +#include "util/defer_op.h" namespace doris { namespace { @@ -910,14 +912,21 @@ void write_struct_filter_parquet_file(const std::string& file_path) { builder.build())); } -void write_dictionary_filter_parquet_file(const std::string& file_path) { +void write_dictionary_filter_parquet_file( + const std::string& file_path, + ::parquet::Compression::type compression = ::parquet::Compression::UNCOMPRESSED) { auto schema = arrow::schema({ arrow::field("id", arrow::int32(), false), arrow::field("value", arrow::utf8(), false), }); - auto table = - arrow::Table::Make(schema, {build_int32_array({1, 2, 3, 4, 5, 6}), - build_string_array({"aa", "az", "lm", "lz", "za", "zz"})}); + const std::vector values = + compression == ::parquet::Compression::UNCOMPRESSED + ? std::vector {"aa", "az", "lm", "lz", "za", "zz"} + : std::vector {std::string(4096, 'a'), std::string(4096, 'b'), + std::string(4096, 'c'), std::string(4096, 'd'), + std::string(4096, 'e'), std::string(4096, 'f')}; + auto table = arrow::Table::Make( + schema, {build_int32_array({1, 2, 3, 4, 5, 6}), build_string_array(values)}); auto file_result = arrow::io::FileOutputStream::Open(file_path); ASSERT_TRUE(file_result.ok()) << file_result.status(); @@ -926,7 +935,7 @@ void write_dictionary_filter_parquet_file(const std::string& file_path) { ::parquet::WriterProperties::Builder builder; builder.version(::parquet::ParquetVersion::PARQUET_2_6); builder.data_page_version(::parquet::ParquetDataPageVersion::V2); - builder.compression(::parquet::Compression::UNCOMPRESSED); + builder.compression(compression); builder.enable_dictionary("value"); builder.disable_dictionary("id"); builder.disable_statistics(); @@ -2514,6 +2523,47 @@ TEST_F(NewParquetReaderTest, PredicateFiltersRowGroupsByDictionary) { EXPECT_EQ(values, std::vector({"lm"})); } +TEST_F(NewParquetReaderTest, DictionaryPruningPublishesColdAndWarmNativePageProfile) { + const double old_cache_threshold = config::parquet_page_cache_decompress_threshold; + config::parquet_page_cache_decompress_threshold = 100.0; + Defer restore_cache_threshold { + [&] { config::parquet_page_cache_decompress_threshold = old_cache_threshold; }}; + write_dictionary_filter_parquet_file(_file_path, ::parquet::Compression::SNAPPY); + + auto open_pruned_reader = [&](RuntimeProfile* profile) { + auto reader = create_reader(0, -1, profile, false, nullptr, std::nullopt, true); + TQueryOptions query_options; + query_options.__set_enable_parquet_file_page_cache(true); + RuntimeState state {query_options, TQueryGlobals()}; + RETURN_IF_ERROR(reader->init(&state)); + std::vector schema; + RETURN_IF_ERROR(reader->get_schema(&schema)); + auto request = std::make_shared(); + request->predicate_columns = {field_projection(1)}; + request->non_predicate_columns = {field_projection(0)}; + request->conjuncts.push_back(create_string_in_conjunct(1, {"not-present"})); + use_schema_order_positions(request.get(), schema); + return reader->open(request); + }; + + RuntimeProfile cold_profile("dictionary_pruning_cold_profile"); + ASSERT_TRUE(open_pruned_reader(&cold_profile).ok()); + for (const auto* counter_name : + {"RowGroupsFilteredByDictionary", "PageReadCount", "PageCacheWriteCount", + "ParsePageHeaderNum", "DecompressCount", "DecodeDictTime"}) { + ASSERT_NE(cold_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(cold_profile.get_counter(counter_name)->value(), 0) << counter_name; + } + + RuntimeProfile warm_profile("dictionary_pruning_warm_profile"); + ASSERT_TRUE(open_pruned_reader(&warm_profile).ok()); + for (const auto* counter_name : {"RowGroupsFilteredByDictionary", "PageReadCount", + "PageCacheHitCount", "ParsePageHeaderNum", "DecodeDictTime"}) { + ASSERT_NE(warm_profile.get_counter(counter_name), nullptr) << counter_name; + EXPECT_GT(warm_profile.get_counter(counter_name)->value(), 0) << counter_name; + } +} + TEST_F(NewParquetReaderTest, DictionaryPredicateFiltersRowsInsideRowGroup) { write_single_row_group_dictionary_filter_parquet_file(_file_path); auto parquet_file_reader = ::parquet::ParquetFileReader::OpenFile(_file_path, false); From 246c11a939ba0e76fc31e0389d61f89ebeda34c3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 08:54:27 +0800 Subject: [PATCH 25/34] [fix](be) address native reader review regressions --- .../data_type_serde/data_type_time_serde.cpp | 48 +++++---- .../delimited_text/delimited_text_reader.cpp | 4 +- be/src/format_v2/parquet/parquet_profile.cpp | 8 +- .../format_v2/parquet/parquet_statistics.cpp | 17 +++- .../data_type_serde_parquet_test.cpp | 89 ++++++++++++++--- .../delimited_text/csv_reader_test.cpp | 2 +- .../delimited_text/text_reader_test.cpp | 30 +++++- be/test/format_v2/orc/orc_reader_test.cpp | 8 +- .../format_v2/parquet/parquet_reader_test.cpp | 2 + .../parquet/parquet_statistics_test.cpp | 99 +++++++++++++++++++ .../hive/test_orc_lazy_mat_profile.groovy | 32 +++--- 11 files changed, 280 insertions(+), 59 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index 062edd756ceacb..7027b1e3397cca 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -67,31 +67,41 @@ class TimeV2ParquetConsumer final : public ParquetFixedValueConsumer { const size_t old_size = _data.size(); _data.resize(old_size + num_values); for (size_t row = 0; row < num_values; ++row) { - int64_t micros; + int64_t raw_value; if (_context.physical_type == ParquetPhysicalType::INT32) { DORIS_CHECK_EQ(value_width, sizeof(int32_t)); - micros = static_cast( - unaligned_load(values + row * sizeof(int32_t))) * - 1000; + raw_value = unaligned_load(values + row * sizeof(int32_t)); } else { DORIS_CHECK(_context.physical_type == ParquetPhysicalType::INT64); DORIS_CHECK_EQ(value_width, sizeof(int64_t)); - micros = unaligned_load(values + row * sizeof(int64_t)); - if (_context.time_unit == ParquetTimeUnit::MILLIS) { - if (micros > std::numeric_limits::max() / 1000 || - micros < std::numeric_limits::min() / 1000) { - if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { - _data[old_size + row] = TimeValue::TimeType(); - continue; - } - _data.resize(old_size); - return Status::DataQualityError( - "Parquet TIME value overflows microseconds"); - } - micros *= 1000; - } else if (_context.time_unit == ParquetTimeUnit::NANOS) { - micros /= 1000; + raw_value = unaligned_load(values + row * sizeof(int64_t)); + } + + int64_t units_per_day; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + units_per_day = 86400000; + } else if (_context.time_unit == ParquetTimeUnit::MICROS) { + units_per_day = 86400000000; + } else { + DORIS_CHECK(_context.time_unit == ParquetTimeUnit::NANOS); + units_per_day = 86400000000000; + } + // Validate the declared carrier before rescaling: truncating nanoseconds first could + // turn a value at or beyond 24:00:00 into an apparently valid TIMEV2 value. + if (raw_value < 0 || raw_value >= units_per_day) { + if (_state != nullptr && _state->mark_conversion_failure(old_size + row)) { + _data[old_size + row] = TimeValue::TimeType(); + continue; } + _data.resize(old_size); + return Status::DataQualityError( + "Parquet TIME value {} is outside the one-day domain", raw_value); + } + int64_t micros = raw_value; + if (_context.time_unit == ParquetTimeUnit::MILLIS) { + micros *= 1000; + } else if (_context.time_unit == ParquetTimeUnit::NANOS) { + micros /= 1000; } // Doris TIMEV2 stores signed microseconds in a double. Splitting into calendar fields // and immediately recombining them is an identity operation with several divisions. diff --git a/be/src/format_v2/delimited_text/delimited_text_reader.cpp b/be/src/format_v2/delimited_text/delimited_text_reader.cpp index 273bb0a0785a75..bad7d9cf7d6d1b 100644 --- a/be/src/format_v2/delimited_text/delimited_text_reader.cpp +++ b/be/src/format_v2/delimited_text/delimited_text_reader.cpp @@ -202,8 +202,10 @@ void DelimitedTextReader::_init_profile() { _profile, "RawLinesRead", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_read_before_filter = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsReadBeforeFilter", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); + // RuntimeProfile counter names are global within one profile, so the format prefix preserves + // independent ownership when Parquet and delimited readers are initialized in either order. _text_profile.rows_filtered_by_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( - _profile, "RowsFilteredByConjunct", TUnit::UNIT, file_scan_profile::FILE_READER, 1); + _profile, "DelimitedRowsFilteredByConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_filtered_by_delete_conjunct = ADD_CHILD_COUNTER_WITH_LEVEL( _profile, "RowsFilteredByDeleteConjunct", TUnit::UNIT, DELIMITED_TEXT_PROFILE, 1); _text_profile.rows_returned = ADD_CHILD_COUNTER_WITH_LEVEL( diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index f197c0ec188858..8432cc9deecdbc 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -102,8 +102,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { ADD_CHILD_COUNTER_WITH_LEVEL(profile, "NestedBatches", TUnit::UNIT, parquet_profile, 1); lazy_read_filtered_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredRowsByLazyRead", TUnit::UNIT, parquet_profile, 1); + // Format-specific counters stay below ParquetReader so subtree consumers cannot silently + // attribute them to a different file format initialized on the same RuntimeProfile. filtered_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FilteredBytes", TUnit::BYTES, - file_scan_profile::FILE_READER, 1); + parquet_profile, 1); raw_rows_read = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RawRowsRead", TUnit::UNIT, parquet_profile, 1); column_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ColumnReadTime", parquet_profile, 1); @@ -111,8 +113,8 @@ void ParquetProfile::init(RuntimeProfile* profile) { parse_footer_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ParseFooterTime", parquet_profile, 1); file_reader_create_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "FileReaderCreateTime", parquet_profile, 1); - open_file_num = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, - file_scan_profile::FILE_READER, 1); + open_file_num = + ADD_CHILD_COUNTER_WITH_LEVEL(profile, "FileNum", TUnit::UNIT, parquet_profile, 1); page_index_read_calls = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PageIndexReadCalls", TUnit::UNIT, parquet_profile, 1); page_index_filter_time = diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index 2bf92987e63e59..1491cc793313e5 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -865,9 +865,22 @@ Status select_row_groups_by_metadata( const int row_group_idx = candidate_row_groups == nullptr ? static_cast(candidate_idx) : (*candidate_row_groups)[candidate_idx]; - DORIS_CHECK(row_group_idx >= 0 && - row_group_idx < static_cast(metadata.row_groups.size())); + if (row_group_idx < 0 || row_group_idx >= static_cast(metadata.row_groups.size())) { + // Candidate ids originate in external split metadata; a corrupt id must not terminate + // the BE while planning an otherwise recoverable file scan. + return Status::Corruption("Invalid Parquet row group candidate {} for {} row groups", + row_group_idx, metadata.row_groups.size()); + } const auto& row_group = metadata.row_groups[row_group_idx]; + if (row_group.num_rows < 0) { + return Status::Corruption("Parquet row group {} has negative row count {}", + row_group_idx, row_group.num_rows); + } + if (row_group.num_rows == 0) { + // Native metadata probes construct positive row ranges; empty groups contribute no + // rows and must be discarded before dictionary, statistics, or Bloom reader setup. + continue; + } ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; if (has_expr_zonemap_filter(request, runtime_state) && check_native_statistics(row_group, file_schema, request, pruning_stats, timezone)) { diff --git a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp index 3ced956af03078..971eb067684c34 100644 --- a/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_parquet_test.cpp @@ -414,19 +414,84 @@ TEST(DataTypeSerDeParquetTest, MaterializesTimestampUnitsAndNegativeEpochDirectl } TEST(DataTypeSerDeParquetTest, MaterializesTimeUnitsDirectly) { - TestParquetDecodeSource source; - source.set_fixed_values({3723456789, -1000001}); - ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, - .logical_type = ParquetLogicalType::TIME, - .time_unit = ParquetTimeUnit::MICROS}; - ParquetMaterializationState state; - DataTypeTimeV2 type(6); - auto column = type.create_column(); + struct TimeUnitCase { + ParquetTimeUnit unit; + int64_t units_per_day; + int64_t expected_last_micros; + }; + const std::vector cases { + {ParquetTimeUnit::MILLIS, 86400000, 86399999000}, + {ParquetTimeUnit::MICROS, 86400000000, 86399999999}, + {ParquetTimeUnit::NANOS, 86400000000000, 86399999999}, + }; - ASSERT_TRUE( - type.get_serde()->read_column_from_parquet(*column, source, context, 2, state).ok()); - EXPECT_EQ(type.to_string(*column, 0), "01:02:03.456789"); - EXPECT_EQ(type.to_string(*column, 1), "-00:00:01.000001"); + for (const auto& test_case : cases) { + SCOPED_TRACE(static_cast(test_case.unit)); + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIME, + .time_unit = test_case.unit}; + DataTypeTimeV2 type(6); + + TestParquetDecodeSource source; + source.set_fixed_values({0, test_case.units_per_day - 1}); + ParquetMaterializationState state; + auto column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*column, source, context, 2, state) + .ok()); + const auto& data = assert_cast(*column).get_data(); + EXPECT_EQ(data[0], 0); + EXPECT_EQ(data[1], test_case.expected_last_micros); + + TestParquetDecodeSource nullable_source; + nullable_source.set_fixed_values({-1, test_case.units_per_day}); + IColumn::Filter null_map; + null_map.resize_fill(2, 0); + ParquetMaterializationState nullable_state; + nullable_state.conversion_failure_null_map = &null_map; + auto nullable_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*nullable_column, nullable_source, context, + 2, nullable_state) + .ok()); + EXPECT_EQ(null_map, IColumn::Filter({1, 1})); + + TestParquetDecodeSource strict_source; + strict_source.set_fixed_values({-1}); + ParquetMaterializationState strict_state; + auto strict_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet(*strict_column, strict_source, context, 1, + strict_state) + .ok()); + EXPECT_EQ(strict_column->size(), 0); + + TestParquetDecodeSource unused_invalid_dictionary; + unused_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day - 1, test_case.units_per_day}, {1, 2}); + auto dictionary_context = context; + dictionary_context.encoding = ParquetValueEncoding::DICTIONARY; + ParquetMaterializationState dictionary_state; + auto dictionary_column = type.create_column(); + ASSERT_TRUE(type.get_serde() + ->read_column_from_parquet(*dictionary_column, + unused_invalid_dictionary, + dictionary_context, 2, dictionary_state) + .ok()); + EXPECT_EQ(dictionary_column->size(), 2); + + TestParquetDecodeSource referenced_invalid_dictionary; + referenced_invalid_dictionary.set_fixed_dictionary( + {-1, 0, test_case.units_per_day}, {0}); + ParquetMaterializationState invalid_dictionary_state; + auto invalid_dictionary_column = type.create_column(); + EXPECT_FALSE(type.get_serde() + ->read_column_from_parquet( + *invalid_dictionary_column, referenced_invalid_dictionary, + dictionary_context, 1, invalid_dictionary_state) + .ok()); + EXPECT_EQ(invalid_dictionary_column->size(), 0); + } } TEST(DataTypeSerDeParquetTest, MaterializesTimestampTzDirectly) { diff --git a/be/test/format_v2/delimited_text/csv_reader_test.cpp b/be/test/format_v2/delimited_text/csv_reader_test.cpp index 0ce99630c7af8b..801187474e1be5 100644 --- a/be/test/format_v2/delimited_text/csv_reader_test.cpp +++ b/be/test/format_v2/delimited_text/csv_reader_test.cpp @@ -446,7 +446,7 @@ TEST_F(CsvV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(file_reader_stats.read_rows, 3); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); diff --git a/be/test/format_v2/delimited_text/text_reader_test.cpp b/be/test/format_v2/delimited_text/text_reader_test.cpp index f5f7309e490ff1..97fe3beb0168cb 100644 --- a/be/test/format_v2/delimited_text/text_reader_test.cpp +++ b/be/test/format_v2/delimited_text/text_reader_test.cpp @@ -41,6 +41,7 @@ #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" #include "format_v2/column_mapper.h" +#include "format_v2/parquet/parquet_profile.h" #include "io/io_common.h" #include "runtime/runtime_profile.h" #include "testutil/desc_tbl_builder.h" @@ -414,7 +415,7 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_NE(_profile.get_counter("DeleteConjunctFilterTime"), nullptr); EXPECT_EQ(counter_value(&_profile, "RawLinesRead"), 3); EXPECT_EQ(counter_value(&_profile, "RowsReadBeforeFilter"), 3); - EXPECT_EQ(counter_value(&_profile, "RowsFilteredByConjunct"), 2); + EXPECT_EQ(counter_value(&_profile, "DelimitedRowsFilteredByConjunct"), 2); EXPECT_EQ(io_ctx->predicate_filtered_rows, 2); EXPECT_EQ(counter_value(&_profile, "RowsFilteredByDeleteConjunct"), 0); EXPECT_EQ(counter_value(&_profile, "RowsReturned"), 1); @@ -423,6 +424,33 @@ TEST_F(TextV2ReaderTest, ProfileCountersTrackReadParseDeserializeAndFilter) { EXPECT_EQ(counter_value(&_profile, "CellsDeserialized"), 6); } +TEST_F(TextV2ReaderTest, FormatProfilesKeepDistinctCountersInBothInitializationOrders) { + auto expect_format_children = [](RuntimeProfile* profile) { + TRuntimeProfileTree tree; + profile->to_thrift(&tree, 3); + ASSERT_FALSE(tree.nodes.empty()); + const auto& children = tree.nodes.front().child_counters_map; + ASSERT_TRUE(children.contains("DelimitedTextReader")); + EXPECT_TRUE(children.at("DelimitedTextReader").contains("DelimitedRowsFilteredByConjunct")); + EXPECT_FALSE(children.at("DelimitedTextReader").contains("RowsFilteredByConjunct")); + ASSERT_TRUE(children.contains("ParquetReader")); + EXPECT_TRUE(children.at("ParquetReader").contains("RowsFilteredByConjunct")); + EXPECT_FALSE(children.at("ParquetReader").contains("DelimitedRowsFilteredByConjunct")); + }; + + RuntimeProfile parquet_first("parquet_first"); + parquet::ParquetProfile parquet_first_counters; + parquet_first_counters.init(&parquet_first); + auto parquet_first_text = create_reader(_file_path, &_params, _slots, &_state, &parquet_first); + expect_format_children(&parquet_first); + + RuntimeProfile text_first("text_first"); + auto text_first_reader = create_reader(_file_path, &_params, _slots, &_state, &text_first); + parquet::ParquetProfile text_first_parquet_counters; + text_first_parquet_counters.init(&text_first); + expect_format_children(&text_first); +} + // Scenario: Hive text has no embedded nested schema, but TableColumnMapper still needs semantic // children for complex table columns. The reader synthesizes ARRAY/MAP/STRUCT children from the // slot type while keeping the top-level local id as the text field ordinal from column_idxs. diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index f2a64e26b92d15..d231d9fd091dae 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -5723,8 +5723,8 @@ TEST_F(NewOrcReaderTest, ClosePublishesOrcLazyStatisticsToRuntimeProfile) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 2); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 2); } TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { @@ -5753,8 +5753,8 @@ TEST_F(NewOrcReaderTest, DisableOrcLazyMaterializationKeepsLazyProfileZero) { ASSERT_EQ(rows, 3); ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("FilteredRowsByLazyRead"), nullptr); - EXPECT_EQ(profile.get_counter("FilteredRowsByLazyRead")->value(), 0); + ASSERT_NE(profile.get_counter("OrcFilteredRowsByLazyRead"), nullptr); + EXPECT_EQ(profile.get_counter("OrcFilteredRowsByLazyRead")->value(), 0); } TEST_F(NewOrcReaderTest, ConditionCacheMissMarksSurvivingGranules) { diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index 03f29b2a8856b8..bc6a2b1948e291 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -2261,6 +2261,8 @@ TEST_F(NewParquetReaderTest, ProfileNestsFormatReaderBelowFileReaderAndRecordsTo EXPECT_TRUE(children.at("ParquetReader").contains("ColumnReadTime")); EXPECT_TRUE(children.at("ParquetReader").contains("RowGroupsReadNum")); EXPECT_TRUE(children.at("ParquetReader").contains("FilteredRowsByGroup")); + EXPECT_TRUE(children.at("ParquetReader").contains("FilteredBytes")); + EXPECT_TRUE(children.at("ParquetReader").contains("FileNum")); } TEST_F(NewParquetReaderTest, ReadMultiPredicateColumnsBeforeExpressionFilter) { diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index afe588478cc390..df486cbc0f3360 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -30,6 +30,7 @@ #include #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" #include "core/field.h" #include "exprs/expr_zonemap_filter.h" #include "exprs/vexpr.h" @@ -103,6 +104,29 @@ class BloomInExpr final : public VExpr { std::vector _values; const std::string _expr_name = "BloomInExpr"; }; + +class DictionaryStringInExpr final : public VExpr { +public: + DictionaryStringInExpr() : VExpr(std::make_shared(), false) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("DictionaryStringInExpr is metadata-only"); + } + + bool can_evaluate_dictionary_filter() const override { return true; } + + ZoneMapFilterResult evaluate_dictionary_filter(const DictionaryEvalContext&) const override { + return ZoneMapFilterResult::kNoMatch; + } + + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + +private: + const std::string _expr_name = "DictionaryStringInExpr"; +}; VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -216,6 +240,81 @@ TEST(ParquetBloomFilterPruningTest, NativeRowGroupKeepsPresentUint32AboveInt32Ma EXPECT_EQ(selected_row_groups, std::vector({0})); EXPECT_EQ(pruning_stats.filtered_row_groups_by_bloom_filter, 0); } + +TEST(NativeParquetStatisticsTest, EmptyDictionaryRowGroupIsSkippedBeforeMetadataProbes) { + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("value"); + leaf.__set_type(tparquet::Type::BYTE_ARRAY); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::BYTE_ARRAY); + column_metadata.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column_metadata.__set_num_values(0); + column_metadata.__set_total_compressed_size(0); + column_metadata.__set_data_page_offset(0); + column_metadata.__set_dictionary_page_offset(0); + column_metadata.__set_encodings({tparquet::Encoding::RLE_DICTIONARY}); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(0); + row_group.__set_num_rows(0); + tparquet::FileMetaData thrift_metadata; + thrift_metadata.__set_version(1); + thrift_metadata.__set_schema({root, leaf}); + thrift_metadata.__set_num_rows(0); + thrift_metadata.__set_row_groups({row_group}); + + format::parquet::NativeParquetMetadata native_metadata(thrift_metadata, 0); + ASSERT_TRUE(native_metadata.init_schema(false, false).ok()); + format::parquet::ParquetFileContext file_context; + file_context.native_file = + std::make_shared(std::vector {}); + file_context.native_metadata = &native_metadata; + + auto column_schema = std::make_unique(); + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::BYTE_ARRAY; + column_schema->type_descriptor.is_string_like = true; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = {VExprContext::create_shared(std::make_shared())}; + std::vector selected_row_groups; + format::parquet::ParquetPruningStats pruning_stats; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata( + thrift_metadata, schema, request, nullptr, &selected_row_groups, true, + &pruning_stats, nullptr, nullptr, &file_context) + .ok()); + EXPECT_TRUE(selected_row_groups.empty()); +} + +TEST(NativeParquetStatisticsTest, InvalidCandidateRowGroupReturnsCorruption) { + tparquet::RowGroup row_group; + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); + format::FileScanRequest request; + const std::vector> schema; + const std::vector candidates {1}; + std::vector selected_row_groups; + + const auto status = format::parquet::select_row_groups_by_metadata( + metadata, schema, request, &candidates, &selected_row_groups, false, nullptr, nullptr, + nullptr, nullptr); + EXPECT_TRUE(status.is()) << status; +} TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrdering) { format::parquet::ParquetTypeDescriptor binary_type; binary_type.physical_type = tparquet::Type::BYTE_ARRAY; diff --git a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy index 98f03c46e48959..981cd0e3b98676 100644 --- a/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy +++ b/regression-test/suites/external_table_p0/hive/test_orc_lazy_mat_profile.groovy @@ -177,25 +177,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("2", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("2", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("1", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("1", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -208,25 +208,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("1", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("3", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -239,25 +239,25 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("0", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("0", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } @@ -271,26 +271,26 @@ suite("test_orc_lazy_mat_profile", "p0,external") { def profileStr = q1() logger.info("profileStr = \n${profileStr}"); - assertEquals("8", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("8", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q2() logger.info("profileStr = \n${profileStr}"); - assertEquals("7", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("7", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q3() logger.info("profileStr = \n${profileStr}"); - assertEquals("6", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("6", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) profileStr = q4() logger.info("profileStr = \n${profileStr}"); - assertEquals("9", extractProfileValue(profileStr, "FilteredRowsByLazyRead")) + assertEquals("9", extractProfileValue(profileStr, "OrcFilteredRowsByLazyRead")) assertEquals("0", extractProfileValue(profileStr, "EvaluatedRowGroupCount")) assertEquals("0", extractProfileValue(profileStr, "SelectedRowGroupCount")) } From 18085f5a663d09c36b8d4861e9a3960fae463fb2 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 12:07:18 +0800 Subject: [PATCH 26/34] [fix](be) optimize nullable Parquet sparse decode --- .../parquet/parquet_column_schema.cpp | 21 +- be/src/format_v2/parquet/parquet_reader.cpp | 58 +++ .../reader/native/column_chunk_reader.cpp | 204 +++++++++ .../reader/native/column_chunk_reader.h | 1 + .../format_v2/parquet/reader/native/decoder.h | 50 ++- be/src/format_v2/table_reader.h | 38 +- .../format_v2/parquet/native_decoder_test.cpp | 404 ++++++++++++++++++ .../format_v2/parquet/parquet_schema_test.cpp | 11 +- be/test/format_v2/table_reader_test.cpp | 41 ++ 9 files changed, 778 insertions(+), 50 deletions(-) diff --git a/be/src/format_v2/parquet/parquet_column_schema.cpp b/be/src/format_v2/parquet/parquet_column_schema.cpp index 53b3bf189e0bc6..610f234a79741c 100644 --- a/be/src/format_v2/parquet/parquet_column_schema.cpp +++ b/be/src/format_v2/parquet/parquet_column_schema.cpp @@ -206,24 +206,6 @@ std::unique_ptr build_native_node_schema(const NativeFieldS return result; } -Status validate_native_node_schema(const NativeFieldSchema& field) { - if (field.children.empty()) { - ParquetTypeDescriptor descriptor; - fill_native_type_descriptor(field, &descriptor); - if (!descriptor.unsupported_reason.empty()) { - // Native metadata must enforce the same logical-type contract used by scan planning; - // otherwise unsupported files reach the decoder only after their schema is accepted. - return Status::NotSupported("Unsupported parquet column '{}': {}", field.name, - descriptor.unsupported_reason); - } - return Status::OK(); - } - for (const auto& child : field.children) { - RETURN_IF_ERROR(validate_native_node_schema(child)); - } - return Status::OK(); -} - } // namespace Status build_parquet_column_schema(const NativeFieldDescriptor& schema, @@ -235,7 +217,8 @@ Status build_parquet_column_schema(const NativeFieldDescriptor& schema, const auto& native_fields = schema.get_fields_schema(); fields->reserve(native_fields.size()); for (size_t field_idx = 0; field_idx < native_fields.size(); ++field_idx) { - RETURN_IF_ERROR(validate_native_node_schema(native_fields[field_idx])); + // Unsupported logical leaves stay in the file schema so request-level validation can + // ignore unprojected fields and COUNT(*) placeholders without weakening real projections. // The scan projection and native readers must share one tree; rebuilding wrappers through // Arrow changes legacy LIST/STRUCT boundaries and makes valid nested values look absent. fields->push_back( diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index ac3db1fd81b83b..93a5ea9bcbabe9 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -57,6 +57,64 @@ struct ParquetReaderScanState { bool enable_strict_mode = false; }; +Status validate_all_projected_leaves_supported(const ParquetColumnSchema& column_schema) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE) { + if (!column_schema.type_descriptor.unsupported_reason.empty()) { + return Status::NotSupported("Unsupported parquet column '{}': {}", column_schema.name, + column_schema.type_descriptor.unsupported_reason); + } + return Status::OK(); + } + for (const auto& child : column_schema.children) { + DORIS_CHECK(child != nullptr); + RETURN_IF_ERROR(validate_all_projected_leaves_supported(*child)); + } + return Status::OK(); +} + +Status validate_projected_leaves_supported(const ParquetColumnSchema& column_schema, + const format::LocalColumnIndex& projection) { + if (column_schema.kind == ParquetColumnSchemaKind::PRIMITIVE || + projection.project_all_children || projection.children.empty()) { + return validate_all_projected_leaves_supported(column_schema); + } + for (const auto& child_projection : projection.children) { + const auto child_it = + std::ranges::find_if(column_schema.children, [&](const auto& child_schema) { + return child_schema->local_id == child_projection.local_id(); + }); + DORIS_CHECK(child_it != column_schema.children.end()); + RETURN_IF_ERROR(validate_projected_leaves_supported(**child_it, child_projection)); + } + return Status::OK(); +} + +Status validate_requested_columns_supported( + const std::vector>& file_schema, + const format::FileScanRequest& request) { + // Validate the projected native-schema leaves before pruning: checking a physical carrier at + // a later read site can let an unsupported logical type silently pass when every row is pruned. + auto validate_scan_column = [&](const format::LocalColumnIndex& projection) -> Status { + const auto local_id = projection.local_id(); + if (local_id == format::ROW_POSITION_COLUMN_ID || + local_id == format::GLOBAL_ROWID_COLUMN_ID) { + return Status::OK(); + } + DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); + DORIS_CHECK(file_schema[local_id] != nullptr); + return validate_projected_leaves_supported(*file_schema[local_id], projection); + }; + for (const auto& column : request.predicate_columns) { + RETURN_IF_ERROR(validate_scan_column(column)); + } + for (const auto& column : request.non_predicate_columns) { + if (!request.is_count_star_placeholder(column.column_id())) { + RETURN_IF_ERROR(validate_scan_column(column)); + } + } + return Status::OK(); +} + const ParquetColumnSchema& projected_root_schema( const std::vector>& file_schema, const format::LocalColumnIndex& projection) { diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 305fe8db2525bd..0b39ba04af67d3 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -31,6 +31,9 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "core/column/column.h" +#include "core/column/column_decimal.h" +#include "core/column/column_dictionary.h" +#include "core/column/column_varbinary.h" #include "core/column/column_vector.h" #include "core/custom_allocator.h" #include "core/data_type_serde/data_type_serde.h" @@ -404,6 +407,197 @@ Status decode_selected_non_null_values(IColumn& column, const DataTypeSerDe& ser state); } +template +bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { +#define VISIT_COLUMN(TYPE) \ + if (auto* typed = check_and_get_column(&column)) { \ + visitor(*typed); \ + return true; \ + } + VISIT_COLUMN(ColumnUInt8) + VISIT_COLUMN(ColumnInt8) + VISIT_COLUMN(ColumnInt16) + VISIT_COLUMN(ColumnInt32) + VISIT_COLUMN(ColumnInt64) + VISIT_COLUMN(ColumnInt128) + VISIT_COLUMN(ColumnDate) + VISIT_COLUMN(ColumnDateTime) + VISIT_COLUMN(ColumnDateV2) + VISIT_COLUMN(ColumnDateTimeV2) + VISIT_COLUMN(ColumnFloat32) + VISIT_COLUMN(ColumnFloat64) + VISIT_COLUMN(ColumnIPv4) + VISIT_COLUMN(ColumnIPv6) + VISIT_COLUMN(ColumnTimeV2) + VISIT_COLUMN(ColumnTimeStampTz) + VISIT_COLUMN(ColumnOffset32) + VISIT_COLUMN(ColumnOffset64) + VISIT_COLUMN(ColumnDecimal32) + VISIT_COLUMN(ColumnDecimal64) + VISIT_COLUMN(ColumnDecimal128V2) + VISIT_COLUMN(ColumnDecimal128V3) + VISIT_COLUMN(ColumnDecimal256) + VISIT_COLUMN(ColumnString) + VISIT_COLUMN(ColumnString64) + VISIT_COLUMN(ColumnVarbinary) + VISIT_COLUMN(ColumnDictI32) +#undef VISIT_COLUMN + return false; +} + +template +void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + auto& data = column.get_data(); + DORIS_CHECK_EQ(data.size(), old_size + compact_values); + data.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + data[old_size + output] = typename ColumnType::value_type {}; + } else { + DORIS_CHECK(source > 0); + --source; + data[old_size + output] = std::move(data[old_size + source]); + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_string_values(ColumnStr& column, size_t old_size, + size_t compact_values, const NullMap& selected_nulls) { + auto& offsets = column.get_offsets(); + DORIS_CHECK_EQ(offsets.size(), old_size + compact_values); + const Offset prefix_end = old_size == 0 ? 0 : offsets[old_size - 1]; + offsets.resize(old_size + selected_nulls.size()); + size_t source = compact_values; + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] == 0) { + DORIS_CHECK(source > 0); + --source; + offsets[old_size + output] = offsets[old_size + source]; + } else { + offsets[old_size + output] = source == 0 ? prefix_end : offsets[old_size + source - 1]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +template +void expand_nullable_values(ColumnType& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_pod_values(column, old_size, compact_values, selected_nulls); +} + +template +void expand_nullable_values(ColumnStr& column, size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + expand_nullable_string_values(column, old_size, compact_values, selected_nulls); +} + +void remap_nullable_conversion_failures(IColumn::Filter* conversion_failure_null_map, + size_t old_size, size_t compact_values, + const NullMap& selected_nulls) { + if (conversion_failure_null_map == nullptr) { + return; + } + DORIS_CHECK(conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + size_t source = compact_values; + // Walk backwards so writing an expanded row cannot overwrite an unread compact failure bit. + for (size_t output = selected_nulls.size(); output > 0;) { + --output; + if (selected_nulls[output] != 0) { + (*conversion_failure_null_map)[old_size + output] = 1; + } else { + DORIS_CHECK(source > 0); + --source; + (*conversion_failure_null_map)[old_size + output] = + (*conversion_failure_null_map)[old_size + source]; + } + } + DORIS_CHECK_EQ(source, 0); +} + +Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& serde, + Decoder& decoder, const ParquetDecodeContext& context, + ParquetMaterializationState& state, + ColumnSelectVector& select_vector, NullMap& selected_nulls, + int64_t* materialization_time) { + auto& selection = state.selection; + selection.ranges.clear(); + selection.total_values = 0; + selection.selected_values = 0; + selected_nulls.clear(); + selected_nulls.reserve(select_vector.num_values() - select_vector.num_filtered()); + + size_t physical_cursor = 0; + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls.resize_fill(selected_nulls.size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls.resize_fill(selected_nulls.size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls.size(), + select_vector.num_values() - select_vector.num_filtered()); + + const size_t old_size = column.size(); + SCOPED_RAW_TIMER(materialization_time); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(decoder.skip_values(selection.total_values)); + column.insert_many_defaults(selected_nulls.size()); + return Status::OK(); + } + + if (state.conversion_failure_null_map != nullptr) { + DORIS_CHECK(state.conversion_failure_null_map->size() >= old_size + selected_nulls.size()); + memset(state.conversion_failure_null_map->data() + old_size, 0, selection.selected_values); + } + SelectedDecodeSource selected_source(decoder, selection); + const auto status = read_native_or_serde(column, serde, selected_source, context, + selection.selected_values, state); + if (!status.ok()) { + if (state.conversion_failure_null_map != nullptr) { + memcpy(state.conversion_failure_null_map->data() + old_size, selected_nulls.data(), + selected_nulls.size()); + } + return status; + } + DORIS_CHECK_EQ(column.size(), old_size + selection.selected_values); + + remap_nullable_conversion_failures(state.conversion_failure_null_map, old_size, + selection.selected_values, selected_nulls); + const bool expanded = visit_nullable_expandable_column(column, [&](auto& typed_column) { + // Decode into the final nested column compactly, then expand in place. This preserves the + // V2 no-intermediate-column invariant while matching StarRocks' nullable sparse layout. + expand_nullable_values(typed_column, old_size, selection.selected_values, selected_nulls); + }); + DORIS_CHECK(expanded); + return Status::OK(); +} + } // namespace template @@ -1068,6 +1262,16 @@ Status ColumnChunkReader::materialize_values( state, select_vector, &_chunk_statistics.materialization_time); _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); + } else if (context.encoding == ParquetValueEncoding::DICTIONARY && + visit_nullable_expandable_column(*doris_column, [](auto&) {})) { + // PLAIN fixed-width pages already skip by pointer arithmetic; compact-and-expand is a + // net loss there. Keep this StarRocks-style nullable batching on dictionary pages, + // where it also collapses thousands of one-value RLE decoder calls into range reads. + ++_chunk_statistics.hybrid_selection_batches; + status = decode_selected_nullable_values( + *doris_column, serde, *_page_decoder, context, state, select_vector, + _nullable_selection_nulls, &_chunk_statistics.materialization_time); + _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); } else { ++_chunk_statistics.hybrid_selection_null_fallback_batches; status = decode_selected_values(*doris_column, serde, *_page_decoder, context, diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index f773b6c1b1bc0f..6bbcebb1bb9bd0 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -344,6 +344,7 @@ class ColumnChunkReader { // Map: encoding -> Decoder // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding std::unordered_map> _decoders; + NullMap _nullable_selection_nulls; ColumnChunkReaderStatistics _chunk_statistics; }; diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index e2ec6a4d4ba9ee..526a6ecfb3590a 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -141,27 +141,32 @@ class BaseDictDecoder : public Decoder { Status decode_selected_dictionary_indices(const ParquetSelection& selection, std::vector* indices) override { DORIS_CHECK(indices != nullptr); - _skip_indices.resize(selection.total_values); - const auto decoded = _index_batch_decoder->GetBatch( - _skip_indices.data(), cast_set(selection.total_values)); - if (UNLIKELY(decoded != selection.total_values)) { - return Status::IOError("Can't read enough Parquet dictionary indices"); - } const size_t num_dictionary_values = dictionary_size(); - for (size_t row = 0; row < selection.total_values; ++row) { - if (UNLIKELY(_skip_indices[row] >= num_dictionary_values)) { - return Status::Corruption( - "Parquet dictionary index {} at row {} exceeds dictionary size {}", - _skip_indices[row], row, num_dictionary_values); - } - } indices->resize(selection.selected_values); + size_t cursor = 0; size_t output = 0; for (const auto& range : selection.ranges) { - memcpy(indices->data() + output, _skip_indices.data() + range.first, - range.count * sizeof(uint32_t)); + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + const auto decoded = _index_batch_decoder->GetBatch(indices->data() + output, + cast_set(range.count)); + if (UNLIKELY(decoded != range.count)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + for (size_t row = 0; row < range.count; ++row) { + if (UNLIKELY((*indices)[output + row] >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + (*indices)[output + row], range.first + row, num_dictionary_values); + } + } output += range.count; + cursor = range.first + range.count; } + DORIS_CHECK(cursor <= selection.total_values); + RETURN_IF_ERROR(_decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values)); DORIS_CHECK_EQ(output, selection.selected_values); return Status::OK(); } @@ -176,16 +181,24 @@ class BaseDictDecoder : public Decoder { protected: Status skip_values(size_t num_values) override { + return _decode_and_validate_skipped(num_values, 0, dictionary_size()); + } + + Status _decode_and_validate_skipped(size_t num_values, size_t row_offset, + size_t num_dictionary_values) { constexpr size_t kSkipBatchSize = 4096; + // Skipped dictionary ids are still external input and must be bounds-checked, but keeping + // only one bounded gap buffer avoids the page-sized scratch used by sparse selections. _skip_indices.resize(std::min(num_values, kSkipBatchSize)); - const size_t num_dictionary_values = dictionary_size(); size_t skipped_values = 0; while (skipped_values < num_values) { const size_t batch_size = std::min(num_values - skipped_values, kSkipBatchSize); const auto skipped = _index_batch_decoder->GetBatch(_skip_indices.data(), static_cast(batch_size)); if (UNLIKELY(skipped != batch_size)) { - return Status::IOError("Can't skip enough Parquet dictionary indices"); + return Status::IOError( + "Can't skip enough Parquet dictionary indices at row {}: {} of {}", + row_offset + skipped_values, skipped, batch_size); } // Filter gaps may be huge RLE runs; validate them without allocating by gap size. for (size_t row = 0; row < batch_size; ++row) { @@ -193,7 +206,8 @@ class BaseDictDecoder : public Decoder { return Status::Corruption( "Parquet dictionary index {} at skipped row {} exceeds dictionary " "size {}", - _skip_indices[row], skipped_values + row, num_dictionary_values); + _skip_indices[row], row_offset + skipped_values + row, + num_dictionary_values); } } skipped_values += batch_size; diff --git a/be/src/format_v2/table_reader.h b/be/src/format_v2/table_reader.h index e87d8c48946deb..6c61acd6c58294 100644 --- a/be/src/format_v2/table_reader.h +++ b/be/src/format_v2/table_reader.h @@ -744,10 +744,11 @@ class TableReader { Status finalize_chunk(Block* block, const size_t rows) { SCOPED_TIMER(_profile.finalize_timer); size_t idx = 0; - for (const auto& mapping : _data_reader.column_mapper->mappings()) { + const auto& mappings = _data_reader.column_mapper->mappings(); + for (const auto& mapping : mappings) { ColumnPtr column; RETURN_IF_ERROR(_materialize_mapping_column(mapping, &_data_reader.block_template, rows, - &column)); + &column, idx + 1 == mappings.size())); block->replace_by_position(idx, IColumn::mutate(std::move(column))); idx++; } @@ -1096,6 +1097,17 @@ class TableReader { return IColumn::mutate(std::move(column)); } + static ColumnPtr _take_and_detach_block_column(Block* block, int position) { + DORIS_CHECK(block != nullptr); + DORIS_CHECK(position >= 0 && position < static_cast(block->columns())); + auto& source = block->get_by_position(position); + ColumnPtr column = source.column; + // The final mapping no longer needs the file block. Release its COW owner before mutate(), + // otherwise nested MAP/STRING columns are deep-copied and a multi-GB payload can OOM. + block->replace_by_position(position, source.type->create_column()); + return _detach_column(std::move(column)); + } + static Status _align_column_nullability(ColumnPtr* column, const DataTypePtr& table_type) { DORIS_CHECK(column != nullptr); DORIS_CHECK(column->get() != nullptr); @@ -1239,7 +1251,8 @@ class TableReader { } Status _materialize_mapping_column(const ColumnMapping& mapping, Block* current_block, - const size_t rows, ColumnPtr* column) { + const size_t rows, ColumnPtr* column, + bool take_projection_result = false) { if (!mapping.is_trivial && mapping.file_local_id.has_value() && !mapping.child_mappings.empty()) { DCHECK(mapping.projection != nullptr); @@ -1252,7 +1265,9 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), *mapping.file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; + ColumnPtr result_column = take_projection_result + ? _take_and_detach_block_column(current_block, res_id) + : current_block->get_by_position(res_id).column; RETURN_IF_ERROR( _materialize_complex_mapping_column(mapping, result_column, rows, column)); return Status::OK(); @@ -1271,8 +1286,12 @@ class TableReader { mapping.table_column_name, mapping.global_index.value(), file_local_id, rows, st.to_string(), mapping.debug_string()); } - ColumnPtr result_column = current_block->get_by_position(res_id).column; - *column = _detach_column(std::move(result_column)); + if (take_projection_result) { + *column = _take_and_detach_block_column(current_block, res_id); + } else { + ColumnPtr result_column = current_block->get_by_position(res_id).column; + *column = _detach_column(std::move(result_column)); + } return Status::OK(); } if (mapping.default_expr != nullptr) { @@ -1645,9 +1664,10 @@ class TableReader { for (size_t column_idx = 0; column_idx < _data_reader.column_mapper->mappings().size(); ++column_idx) { ColumnPtr table_column; - RETURN_IF_ERROR( - _materialize_mapping_column(_data_reader.column_mapper->mappings()[column_idx], - &file_block, 2, &table_column)); + RETURN_IF_ERROR(_materialize_mapping_column( + _data_reader.column_mapper->mappings()[column_idx], &file_block, 2, + &table_column, + column_idx + 1 == _data_reader.column_mapper->mappings().size())); block->replace_by_position(column_idx, std::move(table_column)); } return Status::OK(); diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 56785901df15a0..f32ad23545398d 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -45,6 +45,7 @@ #include "util/block_compression.h" #include "util/coding.h" #include "util/faststring.h" +#include "util/rle_encoding.h" #include "util/thrift_util.h" #include "util/timezone_utils.h" @@ -364,6 +365,409 @@ Status materialize_plain_int96(const std::vector& values, select_vector); } +template +Status materialize_selected_plain_int32(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + bool strict_mode = false) { + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(static_cast(logical_values)); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(int32_t)); + if (!payload.empty()) { + memcpy(payload.data(), physical_values.data(), payload.size()); + } + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.enable_strict_mode = strict_mode; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +template +Status materialize_selected_dictionary_int32( + const std::vector& dictionary, const std::vector& physical_ids, + size_t logical_values, const std::vector& run_length_null_map, + const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics) { + std::vector dictionary_payload(dictionary.size() * sizeof(int32_t)); + memcpy(dictionary_payload.data(), dictionary.data(), dictionary_payload.size()); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + // Real Parquet files place column chunks after the file header. Keep the dictionary offset + // non-zero because zero is the metadata sentinel for "no dictionary page". + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 4); + for (const auto id : physical_ids) { + encoder.Put(id); + } + // Parquet bit-packed dictionary runs declare groups of eight; emit the trailing padding so a + // decoder that buffers the declared run never reads beyond the synthetic page. + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 4; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::INT32; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +Status materialize_selected_dictionary_strings(const std::vector& dictionary, + const std::vector& physical_ids, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics) { + auto dictionary_payload = encode_plain_byte_arrays(dictionary); + tparquet::PageHeader dictionary_header; + dictionary_header.type = tparquet::PageType::DICTIONARY_PAGE; + dictionary_header.__set_compressed_page_size(dictionary_payload.size()); + dictionary_header.__set_uncompressed_page_size(dictionary_payload.size()); + dictionary_header.__isset.dictionary_page_header = true; + dictionary_header.dictionary_page_header.__set_num_values( + static_cast(dictionary.size())); + dictionary_header.dictionary_page_header.__set_encoding(tparquet::Encoding::PLAIN); + std::vector bytes(1, 0); + auto dictionary_page = serialize_page(dictionary_header, dictionary_payload); + bytes.insert(bytes.end(), dictionary_page.begin(), dictionary_page.end()); + const size_t data_page_offset = bytes.size(); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (const auto id : physical_ids) { + encoder.Put(id); + } + for (size_t padding = physical_ids.size(); padding % 8 != 0; ++padding) { + encoder.Put(0); + } + encoder.Flush(); + std::vector data_payload(encoded_ids.size() + 1); + data_payload[0] = 2; + memcpy(data_payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + tparquet::PageHeader data_header; + data_header.type = tparquet::PageType::DATA_PAGE; + data_header.__set_compressed_page_size(data_payload.size()); + data_header.__set_uncompressed_page_size(data_payload.size()); + data_header.__isset.data_page_header = true; + data_header.data_page_header.__set_num_values(static_cast(logical_values)); + data_header.data_page_header.__set_encoding(tparquet::Encoding::RLE_DICTIONARY); + data_header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + data_header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + auto data_page = serialize_page(data_header, data_payload); + bytes.insert(bytes.end(), data_page.begin(), data_page.end()); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::BYTE_ARRAY); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(static_cast(logical_values)); + chunk.meta_data.__set_total_compressed_size(bytes.size() - 1); + chunk.meta_data.__set_dictionary_page_offset(1); + chunk.meta_data.__set_data_page_offset(static_cast(data_page_offset)); + NativeFieldSchema field; + field.physical_type = tparquet::Type::BYTE_ARRAY; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + RETURN_IF_ERROR(chunk_reader.load_page_data()); + + DataTypeString type; + ParquetDecodeContext decode_context; + decode_context.physical_type = ParquetPhysicalType::BYTE_ARRAY; + ParquetMaterializationState state; + state.conversion_failure_null_map = null_map; + FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_values.data(), filter_values.size(), false)); + ColumnSelectVector select_vector; + RETURN_IF_ERROR(select_vector.init(run_length_null_map, logical_values, null_map, &filter, 0)); + RETURN_IF_ERROR(chunk_reader.materialize_values(*column, *type.get_serde(), decode_context, + state, select_vector)); + *statistics = chunk_reader.statistics(); + return Status::OK(); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainSelectionRetainsCheaperRangeFallback) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 0, 500, 0, 1000, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + // Fixed-width PLAIN skips are pointer arithmetic, so compact-and-expand would add work. + EXPECT_EQ(statistics.hybrid_selection_batches, 0); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector dictionary(16); + std::iota(dictionary.begin(), dictionary.end(), 100); + // One value followed by nineteen NULLs exercises StarRocks' sparse-null threshold (<10%). + std::vector null_runs; + for (size_t row = 0; row < LOGICAL_VALUES;) { + null_runs.push_back(1); + ++row; + const auto null_count = std::min(19, LOGICAL_VALUES - row); + null_runs.push_back(cast_set(null_count)); + row += null_count; + } + std::vector physical_ids((LOGICAL_VALUES + 19) / 20); + for (size_t index = 0; index < physical_ids.size(); ++index) { + physical_ids[index] = static_cast(index % dictionary.size()); + } + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeInt32 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(-7); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + const auto status = materialize_selected_dictionary_int32( + dictionary, physical_ids, LOGICAL_VALUES, null_runs, filter, type, &column, &null_map, + &statistics); + ASSERT_TRUE(status.ok()) << status; + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt32::Container {-7, 100, 102, 0, 104, 0})); + EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionRemapsConversionFailures) { + const std::vector dictionary {1, 1000, 2, 3}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeInt8 type; + auto column = type.create_column(); + assert_cast(*column).get_data().push_back(9); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_int32(dictionary, physical_ids, 7, null_runs, + filter, type, &column, &null_map, &statistics) + .ok()); + + EXPECT_EQ(assert_cast(*column).get_data(), + (ColumnInt8::Container {9, 0, 0, 2, 0})); + EXPECT_EQ(null_map, (NullMap {0, 1, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionExpandsStringsInPlace) { + const std::vector dictionary {"a", "bbb", "cc", "dddd"}; + const std::vector physical_ids {0, 1, 2, 3}; + const std::vector null_runs(7, 1); + std::vector filter(7, 0); + for (const size_t selected_row : {1, 2, 4, 5}) { + filter[selected_row] = 1; + } + + DataTypeString type; + auto column = type.create_column(); + assert_cast(*column).insert_data("prefix", 6); + NullMap null_map; + null_map.push_back(0); + ColumnChunkReaderStatistics statistics; + ASSERT_TRUE(materialize_selected_dictionary_strings(dictionary, physical_ids, 7, null_runs, + filter, &column, &null_map, &statistics) + .ok()); + + ASSERT_EQ(column->size(), 5); + EXPECT_EQ(column->get_data_at(0).to_string_view(), "prefix"); + EXPECT_EQ(column->get_data_at(1).to_string_view(), ""); + EXPECT_EQ(column->get_data_at(2).to_string_view(), "bbb"); + EXPECT_EQ(column->get_data_at(3).to_string_view(), "cc"); + EXPECT_EQ(column->get_data_at(4).to_string_view(), ""); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes) { + DataTypeInt32 type; + ColumnChunkReaderStatistics statistics; + + auto all_null_column = type.create_column(); + assert_cast(*all_null_column).get_data().push_back(8); + NullMap all_null_map; + all_null_map.push_back(0); + const std::vector select_alternating {1, 0, 1, 0, 1}; + ASSERT_TRUE(materialize_selected_plain_int32({}, 5, {0, 5}, select_alternating, type, + &all_null_column, &all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 0); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); + + auto dictionary_all_null_column = type.create_column(); + assert_cast(*dictionary_all_null_column).get_data().push_back(8); + NullMap dictionary_all_null_map; + dictionary_all_null_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10}, {}, 5, {0, 5}, select_alternating, type, + &dictionary_all_null_column, + &dictionary_all_null_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_null_column).get_data(), + (ColumnInt32::Container {8, 0, 0, 0})); + EXPECT_EQ(dictionary_all_null_map, (NullMap {0, 1, 1, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); + + auto all_filtered_column = type.create_column(); + assert_cast(*all_filtered_column).get_data().push_back(9); + NullMap all_filtered_map; + all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_plain_int32( + {10, 20, 30}, 5, {1, 1, 1, 1, 1}, std::vector(5, 0), type, + &all_filtered_column, &all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 0); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); + + auto dictionary_all_filtered_column = type.create_column(); + assert_cast(*dictionary_all_filtered_column).get_data().push_back(9); + NullMap dictionary_all_filtered_map; + dictionary_all_filtered_map.push_back(0); + statistics = {}; + ASSERT_TRUE(materialize_selected_dictionary_int32({10, 20, 30}, {0, 1, 2}, 5, {1, 1, 1, 1, 1}, + std::vector(5, 0), type, + &dictionary_all_filtered_column, + &dictionary_all_filtered_map, &statistics) + .ok()); + EXPECT_EQ(assert_cast(*dictionary_all_filtered_column).get_data(), + (ColumnInt32::Container {9})); + EXPECT_EQ(dictionary_all_filtered_map, (NullMap {0})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 0); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + TEST(ParquetV2NativeDecoderTest, ByteArrayDictionaryReferencesOwnedPageAndValidatesIndices) { int32_t dictionary_length = 0; auto dictionary = make_byte_array_dictionary({"alpha", "beta"}, &dictionary_length); diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 4aa0be0fc5adc8..4bebb6da200e16 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -75,7 +75,7 @@ TEST(ParquetSchemaTest, NativeMetadataTreePreservesNestedFieldNamesAndIds) { EXPECT_EQ(mapping->file_child_name("minReaderVersion"), "minReaderVersion"); ASSERT_NE(mapping->child("minReaderVersion"), nullptr); } -TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsRejected) { +TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsDeferredToProjectionValidation) { tparquet::SchemaElement root; root.__set_name("schema"); root.__set_num_children(1); @@ -95,9 +95,12 @@ TEST(ParquetSchemaTest, NativeLogicalUtcTimeIsRejected) { native_schema.assign_ids(); std::vector> fields; const auto status = build_parquet_column_schema(native_schema, &fields); - EXPECT_FALSE(status.ok()); - EXPECT_NE(status.to_string().find("Parquet TIME with isAdjustedToUTC=true is not supported"), - std::string::npos); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(fields.size(), 1); + // Preserve the unsupported marker in metadata; request-level validation decides whether the + // leaf is a real projection or an ignorable COUNT(*) placeholder. + EXPECT_EQ(fields[0]->type_descriptor.unsupported_reason, + "Parquet TIME with isAdjustedToUTC=true is not supported"); } TEST(ParquetSchemaTest, NativeGroupEnumLogicalTypeIsRejected) { diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index d0096990357935..3a6cf401019337 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -327,9 +327,50 @@ class NullableArrayBigintDefaultExpr final : public VExpr { class TableReaderMaterializeTestHelper final : public TableReader { public: + using TableReader::_materialize_mapping_column; using TableReader::_materialize_map_mapping_column; }; +TEST(TableReaderTest, LastProjectionDetachesNestedMapWithoutCopyingStrings) { + auto keys = ColumnString::create(); + keys->insert_data(std::string(1UL << 20, 'k').data(), 1UL << 20); + auto values = ColumnString::create(); + values->insert_data(std::string(1UL << 20, 'v').data(), 1UL << 20); + const auto* original_value_bytes = values->get_chars().data(); + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->insert_value(1); + auto map = ColumnMap::create(std::move(keys), std::move(values), std::move(offsets)); + auto null_map = ColumnUInt8::create(1, 0); + ColumnPtr source = ColumnNullable::create(std::move(map), std::move(null_map)); + + const auto string_type = std::make_shared(); + const auto map_type = make_nullable(std::make_shared(string_type, string_type)); + Block block; + block.insert({source, map_type, "large_map"}); + source.reset(); + + ColumnMapping mapping; + mapping.global_index = GlobalIndex(0); + mapping.table_column_name = "large_map"; + mapping.file_column_name = "large_map"; + mapping.file_local_id = 0; + mapping.file_type = map_type; + mapping.table_type = map_type; + mapping.is_trivial = true; + mapping.projection = + VExprContext::create_shared(VSlotRef::create_shared(0, 0, -1, map_type, "large_map")); + TableReaderMaterializeTestHelper reader; + ColumnPtr detached; + ASSERT_TRUE(reader._materialize_mapping_column(mapping, &block, 1, &detached, + /*take_projection_result=*/true) + .ok()); + EXPECT_EQ(block.get_by_position(0).column->size(), 0); + const auto& detached_nullable = assert_cast(*detached); + const auto& detached_map = assert_cast(detached_nullable.get_nested_column()); + const auto& detached_values = assert_cast(detached_map.get_values()); + EXPECT_EQ(detached_values.get_chars().data(), original_value_bytes); +} + VExprSPtr table_int32_sum_expr(int left_slot_id, int left_column_id, int right_slot_id, int right_column_id) { const auto int_type = std::make_shared(); From 52e71600d5c958ee85d6885811e8c12fbdb2fc87 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 12:21:04 +0800 Subject: [PATCH 27/34] [chore](be) format rebased Hudi V2 test --- be/test/format_v2/table/hudi_reader_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/test/format_v2/table/hudi_reader_test.cpp b/be/test/format_v2/table/hudi_reader_test.cpp index bdda544c93c792..6f9bea89ffe943 100644 --- a/be/test/format_v2/table/hudi_reader_test.cpp +++ b/be/test/format_v2/table/hudi_reader_test.cpp @@ -23,8 +23,8 @@ #include #include -#include #include +#include #include #include #include From 743b7b0aa3d373d1ae9e2d634cd8cb52cd80db48 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 13:42:42 +0800 Subject: [PATCH 28/34] [opt](be) reduce native file scan overhead --- .../data_type_number_serde.cpp | 9 +- .../data_type_serde/parquet_decode_source.h | 43 +++- be/src/format_v2/column_mapper.cpp | 22 ++ be/src/format_v2/file_reader.cpp | 4 + be/src/format_v2/file_reader.h | 8 + .../format_v2/orc/orc_file_input_stream.cpp | 28 ++- be/src/format_v2/parquet/parquet_profile.cpp | 28 +++ be/src/format_v2/parquet/parquet_profile.h | 2 + be/src/format_v2/parquet/parquet_reader.cpp | 22 +- be/src/format_v2/parquet/parquet_scan.cpp | 224 +++++++++++++++--- be/src/format_v2/parquet/parquet_scan.h | 21 +- .../format_v2/parquet/parquet_statistics.cpp | 83 +++++-- be/src/format_v2/parquet/parquet_statistics.h | 17 +- .../reader/native/column_chunk_reader.cpp | 17 +- .../format_v2/parquet/reader/native/decoder.h | 62 ++++- be/test/format_v2/column_mapper_test.cpp | 2 + .../orc/orc_file_input_stream_test.cpp | 47 +++- be/test/format_v2/orc/orc_reader_test.cpp | 21 +- .../format_v2/parquet/native_decoder_test.cpp | 71 +++++- .../format_v2/parquet/parquet_reader_test.cpp | 10 +- .../format_v2/parquet/parquet_scan_test.cpp | 72 +++++- .../parquet/parquet_statistics_test.cpp | 129 +++++++++- be/test/format_v2/table_reader_test.cpp | 1 + 23 files changed, 808 insertions(+), 135 deletions(-) diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 20d7771209a4e5..783884a073f085 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -242,8 +242,7 @@ Status append_parquet_number(PaddedPODArray& data, const uint8_t* if (!decoded_number_value_fits(value)) { if (state != nullptr && state->can_insert_null_on_conversion_failure()) { data[old_size + row] = DorisCppType(); - DORIS_CHECK_LT(old_size + row, state->conversion_failure_null_map->size()); - (*state->conversion_failure_null_map)[old_size + row] = 1; + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); continue; } data.resize(old_size); @@ -274,8 +273,7 @@ Status append_parquet_logical_integers(PaddedPODArray& data, const if (!decoded_number_value_fits(logical_value)) { if (state != nullptr && state->can_insert_null_on_conversion_failure()) { data[old_size + row] = DorisCppType(); - DORIS_CHECK_LT(old_size + row, state->conversion_failure_null_map->size()); - (*state->conversion_failure_null_map)[old_size + row] = 1; + DORIS_CHECK(state->mark_conversion_failure(old_size + row)); continue; } data.resize(old_size); @@ -373,8 +371,7 @@ class NumberParquetConsumer final : public ParquetFixedValueConsumer { if (!decoded_number_value_fits(value)) { if (_state != nullptr && _state->can_insert_null_on_conversion_failure()) { _data[old_size + row] = DorisCppType(); - DORIS_CHECK_LT(old_size + row, _state->conversion_failure_null_map->size()); - (*_state->conversion_failure_null_map)[old_size + row] = 1; + DORIS_CHECK(_state->mark_conversion_failure(old_size + row)); continue; } _data.resize(old_size); diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index 1c56f4b10e4769..7055051dc19923 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -233,12 +233,16 @@ struct ParquetMaterializationState { IColumn::Filter* conversion_failure_null_map = nullptr; IColumn::Filter dictionary_conversion_failures; bool capturing_dictionary_conversion_failures = false; + bool dictionary_has_conversion_failures = false; + size_t dictionary_failure_scan_rows = 0; void reset_dictionary() { typed_dictionary.reset(); dictionary_indices.clear(); dictionary_conversion_failures.clear(); capturing_dictionary_conversion_failures = false; + dictionary_has_conversion_failures = false; + dictionary_failure_scan_rows = 0; dictionary_generation = std::numeric_limits::max(); } @@ -253,12 +257,16 @@ struct ParquetMaterializationState { } DORIS_CHECK_LT(output_row, conversion_failure_null_map->size()); (*conversion_failure_null_map)[output_row] = 1; + if (capturing_dictionary_conversion_failures) { + dictionary_has_conversion_failures = true; + } return true; } IColumn::Filter* begin_dictionary_conversion(size_t dictionary_size) { auto* output_null_map = conversion_failure_null_map; dictionary_conversion_failures.resize_fill(dictionary_size, 0); + dictionary_has_conversion_failures = false; conversion_failure_null_map = &dictionary_conversion_failures; capturing_dictionary_conversion_failures = true; return output_null_map; @@ -271,23 +279,34 @@ struct ParquetMaterializationState { Status materialize_dictionary(IColumn& column) { const size_t old_size = column.size(); - for (size_t row = 0; row < dictionary_indices.size(); ++row) { - const auto dictionary_id = dictionary_indices[row]; - DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); - if (dictionary_conversion_failures[dictionary_id] != 0 && - !can_insert_null_on_conversion_failure()) { - // A malformed dictionary entry is irrelevant until a selected row references it; - // failing while building the dictionary would reject otherwise valid pages. - return Status::DataQualityError( - "Parquet dictionary entry {} cannot be converted to the target type", - dictionary_id); + dictionary_failure_scan_rows = 0; + if (UNLIKELY(dictionary_has_conversion_failures)) { + const bool insert_failure_as_null = can_insert_null_on_conversion_failure(); + if (!insert_failure_as_null) { + for (size_t row = 0; row < dictionary_indices.size(); ++row) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] == 0) { + continue; + } + // A malformed dictionary entry is irrelevant until a selected row references + // it; failing while building the dictionary would reject valid pages. + return Status::DataQualityError( + "Parquet dictionary entry {} cannot be converted to the target type", + dictionary_id); + } } } column.insert_indices_from(*typed_dictionary, dictionary_indices.data(), dictionary_indices.data() + dictionary_indices.size()); - if (can_insert_null_on_conversion_failure()) { + if (UNLIKELY(dictionary_has_conversion_failures && + can_insert_null_on_conversion_failure())) { for (size_t row = 0; row < dictionary_indices.size(); ++row) { - if (dictionary_conversion_failures[dictionary_indices[row]] != 0) { + ++dictionary_failure_scan_rows; + const auto dictionary_id = dictionary_indices[row]; + DORIS_CHECK_LT(dictionary_id, dictionary_conversion_failures.size()); + if (dictionary_conversion_failures[dictionary_id] != 0) { mark_conversion_failure(old_size + row); } } diff --git a/be/src/format_v2/column_mapper.cpp b/be/src/format_v2/column_mapper.cpp index 6713183c09d816..4b8f94ae2722db 100644 --- a/be/src/format_v2/column_mapper.cpp +++ b/be/src/format_v2/column_mapper.cpp @@ -2079,6 +2079,7 @@ Status TableColumnMapper::create_scan_request( // table-column to file-column conversion, so it also owns the file-local block positions. file_request->predicate_columns.clear(); file_request->non_predicate_columns.clear(); + file_request->predicate_only_columns.clear(); file_request->local_positions.clear(); file_request->conjuncts.clear(); file_request->delete_conjuncts.clear(); @@ -2110,6 +2111,27 @@ Status TableColumnMapper::create_scan_request( // Hidden filter mappings must be built before localizing filters, so that they can be localized together with visible mappings and referenced by localized filter expressions. RETURN_IF_ERROR(_build_hidden_filter_mappings(table_filters)); RETURN_IF_ERROR(localize_filters(table_filters, file_request, runtime_state)); + for (const auto& mapping : _hidden_mappings) { + if (!mapping.file_local_id.has_value()) { + continue; + } + const auto local_id = LocalColumnId(*mapping.file_local_id); + const bool is_visible_output = + std::ranges::any_of(_mappings, [local_id](const ColumnMapping& visible_mapping) { + return visible_mapping.file_local_id.has_value() && + LocalColumnId(*visible_mapping.file_local_id) == local_id; + }); + if (is_visible_output) { + continue; + } + if (std::ranges::any_of(file_request->predicate_columns, + [local_id](const LocalColumnIndex& projection) { + return projection.column_id() == local_id; + }) && + !file_request->is_predicate_only(local_id)) { + file_request->predicate_only_columns.push_back(local_id); + } + } // 3. Rebuild output projection expressions for projected columns. localize_filters() has // already applied the final scan projection to mapping.file_type/projected_file_children before // rewriting filter expressions. diff --git a/be/src/format_v2/file_reader.cpp b/be/src/format_v2/file_reader.cpp index b2603a17958eae..9bbf7c3a8fc066 100644 --- a/be/src/format_v2/file_reader.cpp +++ b/be/src/format_v2/file_reader.cpp @@ -53,6 +53,10 @@ std::string FileScanRequest::debug_string() const { << join_debug_strings( non_predicate_columns, [](const LocalColumnIndex& projection) { return projection.debug_string(); }) + << ", predicate_only_columns=" + << join_debug_strings( + predicate_only_columns, + [](LocalColumnId column_id) { return std::to_string(column_id.value()); }) << ", local_positions={"; size_t position_idx = 0; for (const auto& [column_id, block_position] : local_positions) { diff --git a/be/src/format_v2/file_reader.h b/be/src/format_v2/file_reader.h index 96d940702486b4..5f959c3e672dcd 100644 --- a/be/src/format_v2/file_reader.h +++ b/be/src/format_v2/file_reader.h @@ -70,6 +70,10 @@ struct FileScanRequest { // Columns read after row-level filtering. Predicate columns are also available for output and // should not be duplicated here. std::vector non_predicate_columns; + // Predicate columns introduced only to evaluate hidden filter slots. Their values are dead + // after all file-local predicates run, although the shared file block still needs row-shaped + // placeholders until TableReader finalizes projected columns. + std::vector predicate_only_columns; // file-local column id -> file-local output block position. std::map local_positions; // Row-level filters converted to file-local expressions from table-level predicates. @@ -89,6 +93,10 @@ struct FileScanRequest { return std::ranges::find(count_star_placeholder_columns, column_id) != count_star_placeholder_columns.end(); } + + bool is_predicate_only(LocalColumnId column_id) const { + return std::ranges::find(predicate_only_columns, column_id) != predicate_only_columns.end(); + } }; // Helper for constructing the scan-column layout in FileScanRequest. diff --git a/be/src/format_v2/orc/orc_file_input_stream.cpp b/be/src/format_v2/orc/orc_file_input_stream.cpp index 303a65c4947bae..df0927aac3444a 100644 --- a/be/src/format_v2/orc/orc_file_input_stream.cpp +++ b/be/src/format_v2/orc/orc_file_input_stream.cpp @@ -59,22 +59,24 @@ class OrcMergedRangeFileReader final : public io::FileReader { _total_time = ADD_CHILD_TIMER_WITH_LEVEL( _profile, profile_name, file_scan_profile::parent_or_root(_profile, file_scan_profile::IO), 1); - _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "CopyTime", profile_name, 1); - _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "ReadTime", profile_name, 1); - _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestIO", TUnit::UNIT, + // RuntimeProfile counter lookup is flat, so every child must be format-qualified; + // a unique parent alone cannot prevent aliasing with Parquet's MergeRange reader. + _copy_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedCopyTime", profile_name, 1); + _read_time = ADD_CHILD_TIMER_WITH_LEVEL(_profile, "OrcMergedReadTime", profile_name, 1); + _request_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestIO", TUnit::UNIT, profile_name, 1); - _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedIO", TUnit::UNIT, + _merged_io = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedIO", TUnit::UNIT, profile_name, 1); - _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "RequestBytes", TUnit::BYTES, - profile_name, 1); - _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "MergedBytes", TUnit::BYTES, + _request_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedRequestBytes", + TUnit::BYTES, profile_name, 1); + _merged_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedBytes", TUnit::BYTES, profile_name, 1); - _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ApplyBytes", TUnit::BYTES, - profile_name, 1); - _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OverReadBytes", TUnit::BYTES, - profile_name, 1); - _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "ClusterNum", TUnit::UNIT, - profile_name, 1); + _apply_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedApplyBytes", + TUnit::BYTES, profile_name, 1); + _over_read_bytes = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedOverReadBytes", + TUnit::BYTES, profile_name, 1); + _cluster_num = ADD_CHILD_COUNTER_WITH_LEVEL(_profile, "OrcMergedClusterNum", + TUnit::UNIT, profile_name, 1); } } diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 8432cc9deecdbc..884a264cefd2cf 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -225,6 +225,34 @@ void ParquetProfile::update_pruning_stats(const ParquetPruningStats& pruning_sta COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); } +void ParquetProfile::update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const { + const int64_t filtered = selected ? 0 : 1; + COUNTER_UPDATE(filtered_row_groups, filtered); + COUNTER_UPDATE(filtered_row_groups_by_dictionary, + pruning_stats.filtered_row_groups_by_dictionary); + COUNTER_UPDATE(filtered_row_groups_by_bloom_filter, + pruning_stats.filtered_row_groups_by_bloom_filter); + COUNTER_UPDATE(filtered_row_groups_by_page_index, + pruning_stats.filtered_row_groups_by_page_index); + // Initial footer planning counts every surviving candidate as readable. Lazy probes correct + // that estimate only when a later dictionary, Bloom, or page-index check removes the group. + COUNTER_UPDATE(to_read_row_groups, -filtered); + COUNTER_UPDATE(selected_row_ranges, pruning_stats.selected_row_ranges); + COUNTER_UPDATE(filtered_group_rows, pruning_stats.filtered_group_rows); + COUNTER_UPDATE(filtered_bytes, pruning_stats.filtered_bytes); + COUNTER_UPDATE(filtered_page_rows, pruning_stats.filtered_page_rows); + COUNTER_UPDATE(page_index_read_calls, pruning_stats.page_index_read_calls); + COUNTER_UPDATE(bloom_filter_read_time, pruning_stats.bloom_filter_read_time); + COUNTER_UPDATE(row_group_filter_time, pruning_stats.row_group_filter_time); + COUNTER_UPDATE(page_index_filter_time, pruning_stats.page_index_filter_time); + COUNTER_UPDATE(read_page_index_time, pruning_stats.read_page_index_time); + COUNTER_UPDATE(parse_page_index_time, pruning_stats.parse_page_index_time); + COUNTER_UPDATE(expr_zonemap_unusable, pruning_stats.expr_zonemap_unusable_evals); + COUNTER_UPDATE(in_zonemap_point_check, pruning_stats.in_zonemap_point_check_count); + COUNTER_UPDATE(in_zonemap_range_only, pruning_stats.in_zonemap_range_only_count); +} + ParquetPageSkipProfile ParquetProfile::page_skip_profile() const { return { .skipped_pages = pages_skipped_by_data_page_filter, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index afd77c8996e614..a225a352296c4b 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -109,6 +109,8 @@ struct ParquetScanProfile { struct ParquetProfile { void init(RuntimeProfile* profile); void update_pruning_stats(const ParquetPruningStats& pruning_stats) const; + void update_deferred_pruning_stats(const ParquetPruningStats& pruning_stats, + bool selected) const; ParquetPageSkipProfile page_skip_profile() const; ParquetColumnReaderProfile column_reader_profile() const; diff --git a/be/src/format_v2/parquet/parquet_reader.cpp b/be/src/format_v2/parquet/parquet_reader.cpp index 93a5ea9bcbabe9..758ff8c5a1490b 100644 --- a/be/src/format_v2/parquet/parquet_reader.cpp +++ b/be/src/format_v2/parquet/parquet_reader.cpp @@ -465,6 +465,9 @@ Status ParquetReader::open(std::shared_ptr request) { // column chunks here: footer offsets are untrusted and this obsolete range map is not consumed. _state->scan_plan = row_group_plan; _state->scheduler.set_page_skip_profile(_parquet_profile.page_skip_profile()); + if (_profile != nullptr) { + _state->scheduler.set_pruning_profile(&_parquet_profile); + } _state->scheduler.set_global_rowid_context(_global_rowid_context); _state->scheduler.set_scan_profile(_parquet_profile.scan_profile()); _state->scheduler.set_plan(std::move(row_group_plan)); @@ -589,6 +592,14 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r request.agg_type); } + // Aggregate pushdown bypasses the scheduler but still requires the exact pruned row-group set. + // Finish lazy remote probes here; normal scans keep them at current-row-group granularity. + RETURN_IF_ERROR(finalize_parquet_row_group_plans( + *_state->file_context.native_metadata, _state->file_schema, *_request, + _state->enable_bloom_filter, &_state->scan_plan, _state->timezone, + _state->runtime_state, &_state->file_context, _parquet_profile.column_reader_profile(), + _profile == nullptr ? nullptr : &_parquet_profile)); + for (const auto& aggregate_column : request.columns) { const auto local_id = aggregate_column.projection.local_id(); if (local_id < 0 || local_id >= static_cast(_state->file_schema.size())) { @@ -697,9 +708,16 @@ Status ParquetReader::get_aggregate_result(const format::FileAggregateRequest& r const auto& column_chunk = row_group_metadata.columns[leaf_schema->leaf_column_id]; DORIS_CHECK(column_chunk.__isset.meta_data); const auto& column_metadata = column_chunk.meta_data; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + leaf_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order( + _state->file_context.native_metadata->to_thrift(), + leaf_schema->leaf_column_id)); + } const auto statistics = ParquetStatisticsUtils::TransformColumnStatistics( - *leaf_schema, - column_metadata.__isset.statistics ? &column_metadata.statistics : nullptr, + *leaf_schema, safe_statistics.has_value() ? &*safe_statistics : nullptr, column_metadata.num_values, _state->timezone); if (!statistics.has_min_max) { return Status::NotSupported("Missing parquet min/max statistics for column {}", diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 5326a22de7fc67..c9d4c0380c674d 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -383,15 +383,55 @@ Status select_native_row_groups_by_scan_range(const tparquet::FileMetaData& meta namespace { -Status build_native_row_group_read_plans( +std::vector intersect_row_ranges(const std::vector& left, + const std::vector& right) { + std::vector result; + size_t left_idx = 0; + size_t right_idx = 0; + while (left_idx < left.size() && right_idx < right.size()) { + const int64_t left_end = left[left_idx].start + left[left_idx].length; + const int64_t right_end = right[right_idx].start + right[right_idx].length; + const int64_t start = std::max(left[left_idx].start, right[right_idx].start); + const int64_t end = std::min(left_end, right_end); + if (start < end) { + result.push_back({.start = start, .length = end - start}); + } + if (left_end < right_end) { + ++left_idx; + } else { + ++right_idx; + } + } + return result; +} + +Status finalize_native_row_group_read_plan( const NativeParquetMetadata& metadata, const std::vector>& file_schema, - const format::FileScanRequest& request, const std::vector& selected_row_groups, - const std::vector& row_group_first_rows, RowGroupScanPlan* plan, + const format::FileScanRequest& request, bool enable_bloom_filter, + RowGroupReadPlan* row_group_plan, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState* runtime_state, - ParquetFileContext* file_context) { - DORIS_CHECK(plan != nullptr && file_context != nullptr); + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + bool* selected) { + DORIS_CHECK(row_group_plan != nullptr && pruning_stats != nullptr && file_context != nullptr && + selected != nullptr); + *selected = true; + if (!row_group_plan->expensive_pruning_pending) { + return Status::OK(); + } + row_group_plan->expensive_pruning_pending = false; const auto& thrift = metadata.to_thrift(); + const std::vector candidate {row_group_plan->row_group_id}; + std::vector metadata_selected; + RETURN_IF_ERROR(select_row_groups_by_metadata( + thrift, file_schema, request, &candidate, &metadata_selected, enable_bloom_filter, + pruning_stats, timezone, runtime_state, file_context, column_reader_profile, + ParquetMetadataProbeMode::EXPENSIVE_ONLY)); + if (metadata_selected.empty()) { + *selected = false; + return Status::OK(); + } + std::unordered_set requested_leaf_ids; for (const auto& projection : request_scan_columns(request)) { const auto local_id = projection.local_id(); @@ -400,6 +440,40 @@ Status build_native_row_group_read_plans( } collect_projected_leaf_column_ids(*file_schema[local_id], projection, &requested_leaf_ids); } + std::unordered_map page_indexes; + if (can_use_parquet_page_index(request, runtime_state)) { + RETURN_IF_ERROR(file_context->load_native_page_indexes( + row_group_plan->row_group_id, requested_leaf_ids, &page_indexes, + &pruning_stats->read_page_index_time, &pruning_stats->parse_page_index_time)); + } + std::vector page_selected_ranges; + std::map page_skip_plans; + RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( + thrift, page_indexes, file_schema, request, row_group_plan->row_group_rows, + &page_selected_ranges, &page_skip_plans, pruning_stats, timezone, runtime_state)); + row_group_plan->selected_ranges = + intersect_row_ranges(row_group_plan->selected_ranges, page_selected_ranges); + row_group_plan->page_skip_plans = std::move(page_skip_plans); + for (auto& [leaf_column_id, indexes] : page_indexes) { + row_group_plan->offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); + } + if (row_group_plan->selected_ranges.empty()) { + *selected = false; + return Status::OK(); + } + pruning_stats->selected_row_ranges += row_group_plan->selected_ranges.size(); + return Status::OK(); +} + +Status build_native_row_group_read_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, const std::vector& selected_row_groups, + const std::vector& row_group_first_rows, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + const auto& thrift = metadata.to_thrift(); plan->row_groups.reserve(selected_row_groups.size()); for (const int row_group_idx : selected_row_groups) { const auto& row_group = thrift.row_groups[row_group_idx]; @@ -410,24 +484,8 @@ Status build_native_row_group_read_plans( row_group_plan.row_group_id = row_group_idx; row_group_plan.first_file_row = row_group_first_rows[row_group_idx]; row_group_plan.row_group_rows = row_group.num_rows; - std::unordered_map page_indexes; - if (can_use_parquet_page_index(request, runtime_state)) { - RETURN_IF_ERROR(file_context->load_native_page_indexes( - row_group_idx, requested_leaf_ids, &page_indexes, - &plan->pruning_stats.read_page_index_time, - &plan->pruning_stats.parse_page_index_time)); - } - RETURN_IF_ERROR(select_row_group_ranges_by_native_page_index( - page_indexes, file_schema, request, row_group.num_rows, - &row_group_plan.selected_ranges, &row_group_plan.page_skip_plans, - &plan->pruning_stats, timezone, runtime_state)); - for (auto& [leaf_column_id, indexes] : page_indexes) { - row_group_plan.offset_indexes.emplace(leaf_column_id, std::move(indexes.offset_index)); - } - if (row_group_plan.selected_ranges.empty()) { - continue; - } - plan->pruning_stats.selected_row_ranges += row_group_plan.selected_ranges.size(); + row_group_plan.selected_ranges = {{.start = 0, .length = row_group.num_rows}}; + row_group_plan.expensive_pruning_pending = true; plan->row_groups.push_back(std::move(row_group_plan)); } return Status::OK(); @@ -445,6 +503,7 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, DORIS_CHECK(plan != nullptr && file_context != nullptr); plan->row_groups.clear(); plan->pruning_stats = {}; + plan->enable_bloom_filter = enable_bloom_filter; std::vector row_group_first_rows; std::vector scan_range_selected; RETURN_IF_ERROR(detail::select_native_row_groups_by_scan_range( @@ -453,7 +512,7 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, RETURN_IF_ERROR(select_row_groups_by_metadata( metadata.to_thrift(), file_schema, request, &scan_range_selected, &metadata_selected, enable_bloom_filter, &plan->pruning_stats, timezone, runtime_state, file_context, - column_reader_profile)); + column_reader_profile, ParquetMetadataProbeMode::FOOTER_ONLY)); RETURN_IF_ERROR(build_native_row_group_read_plans(metadata, file_schema, request, metadata_selected, row_group_first_rows, plan, timezone, runtime_state, file_context)); @@ -461,6 +520,35 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, return Status::OK(); } +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile) { + DORIS_CHECK(plan != nullptr && file_context != nullptr); + std::vector selected_plans; + selected_plans.reserve(plan->row_groups.size()); + for (auto& row_group_plan : plan->row_groups) { + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + metadata, file_schema, request, enable_bloom_filter, &row_group_plan, + &deferred_stats, timezone, runtime_state, file_context, column_reader_profile, + &selected)); + if (parquet_profile != nullptr) { + parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (selected) { + selected_plans.push_back(std::move(row_group_plan)); + } + } + plan->row_groups = std::move(selected_plans); + plan->pruning_stats.selected_row_groups = plan->row_groups.size(); + return Status::OK(); +} + namespace { using DictionaryResidualConjunct = std::pair; @@ -718,6 +806,7 @@ std::vector filter_ranges_by_condition_cache(const std::vector>& file_schema, const format::FileScanRequest& request, bool* has_row_group) { *has_row_group = false; - if (_next_row_group_plan_idx >= _row_group_plans.size()) { + RowGroupReadPlan* selected_plan = nullptr; + while (_next_row_group_plan_idx < _row_group_plans.size()) { + RowGroupReadPlan& candidate_plan = _row_group_plans[_next_row_group_plan_idx++]; + // Probe only the row group about to execute. This keeps LIMIT/cancellation latency + // independent of the number of later remote row groups while preserving eager footer + // statistics pruning during open. + file_context.reset_random_access_ranges(); + _current_merge_range_active = false; + ParquetPruningStats deferred_stats; + bool selected = false; + RETURN_IF_ERROR(finalize_native_row_group_read_plan( + *file_context.native_metadata, file_schema, request, _enable_bloom_filter, + &candidate_plan, &deferred_stats, _timezone, _runtime_state, &file_context, + _scan_profile.column_reader_profile, &selected)); + if (_parquet_profile != nullptr) { + _parquet_profile->update_deferred_pruning_stats(deferred_stats, selected); + } + if (!selected) { + continue; + } + selected_plan = &candidate_plan; + break; + } + if (selected_plan == nullptr) { // The last row group's native readers have already been released by // reset_current_row_group(). Flush the shared merge reader now so its counters are visible // when EOF is returned and its bounded scratch does not survive until file close. @@ -885,7 +997,7 @@ Status ParquetScanScheduler::open_next_row_group( _current_merge_range_active = false; return Status::OK(); } - RowGroupReadPlan& row_group_plan = _row_group_plans[_next_row_group_plan_idx++]; + RowGroupReadPlan& row_group_plan = *selected_plan; const int row_group_idx = row_group_plan.row_group_id; // Dictionary probes and data-page readers share the native metadata tree. Reset the previous // row-group merge reader before probing because dictionary-page offsets are not scan ordered. @@ -1323,7 +1435,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } }; - auto compact_predicate_columns = [&]() -> Status { + auto compact_predicate_columns = [&](bool discard_predicate_only_payload) -> Status { bool compacted = false; int64_t compacted_bytes = 0; for (const uint32_t position : read_column_positions) { @@ -1334,6 +1446,24 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, "Predicate column {} has {} values but {} remembered source rows", position, old_column->size(), source_rows.size()); } + bool predicate_only = false; + if (discard_predicate_only_payload) { + predicate_only = std::ranges::any_of( + request.predicate_only_columns, [&](format::LocalColumnId local_id) { + const auto position_it = request.local_positions.find(local_id); + return position_it != request.local_positions.end() && + position_it->second.value() == position; + }); + } + if (predicate_only) { + auto placeholder = old_column->clone_empty(); + // Hidden predicate values are dead after the last filter, but every file-block + // column must retain the selected row count until TableReader drops hidden slots. + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(position, std::move(placeholder)); + remember_column_selection(position); + continue; + } bool already_compact = source_rows.size() == *selected_rows && old_column->size() == static_cast(*selected_rows); for (uint16_t row = 0; already_compact && row < *selected_rows; ++row) { @@ -1632,9 +1762,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; - auto compact_predicate_columns_with_profile = [&]() -> Status { + auto compact_predicate_columns_with_profile = + [&](bool discard_predicate_only_payload) -> Status { const int64_t start_ns = MonotonicNanos(); - auto status = compact_predicate_columns(); + auto status = compact_predicate_columns(discard_predicate_only_payload); update_counter_if_not_null(_scan_profile.predicate_compaction_time, MonotonicNanos() - start_ns); return status; @@ -1642,17 +1773,23 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, RETURN_IF_ERROR(read_round_by_round()); // Single-column expressions only touch the just-read column, so earlier columns can retain - // their own row mappings. Compact once before a multi-column/output boundary. - RETURN_IF_ERROR(compact_predicate_columns_with_profile()); + // their own row mappings. Compact only when a later expression needs a shared coordinate + // space; otherwise the final boundary can discard hidden predicate payloads without scanning + // them again. + if (!schedule.remaining_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); + } RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(schedule.remaining_conjuncts)); - RETURN_IF_ERROR(compact_predicate_columns_with_profile()); + if (!request.delete_conjuncts.empty()) { + RETURN_IF_ERROR(compact_predicate_columns_with_profile(false)); + } if (_scan_profile.predicate_filter_time == nullptr) { RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } else { SCOPED_TIMER(_scan_profile.predicate_filter_time); RETURN_IF_ERROR(execute_scheduled_delete_conjuncts()); } - return compact_predicate_columns_with_profile(); + return compact_predicate_columns_with_profile(true); } Status ParquetScanScheduler::prefetch_current_row_group_columns( @@ -1847,6 +1984,21 @@ Status ParquetScanScheduler::read_next_batch( const std::vector>& file_schema, const format::FileScanRequest& request, Block* file_block, size_t* rows, bool* eof) { *rows = 0; + int64_t predicate_batch_rows = _batch_size; + const int64_t max_predicate_batch_rows = std::min( + std::numeric_limits::max(), + std::max(DEFAULT_READ_BATCH_SIZE, _runtime_state == nullptr + ? DEFAULT_READ_BATCH_SIZE + : _runtime_state->batch_size())); + auto grow_empty_predicate_batch = [max_predicate_batch_rows](int64_t current) { + for (const int64_t target : + {int64_t {256}, int64_t {1024}, int64_t {4096}, max_predicate_batch_rows}) { + if (current < target) { + return std::min(target, max_predicate_batch_rows); + } + } + return max_predicate_batch_rows; + }; while (true) { if (!_has_current_row_group) { bool has_row_group = false; @@ -1883,7 +2035,7 @@ Status ParquetScanScheduler::read_next_batch( continue; } - const int64_t batch_rows = std::min(_batch_size, remaining_rows); + const int64_t batch_rows = std::min(predicate_batch_rows, remaining_rows); const int64_t physical_rows_read = batch_rows; const int64_t batch_first_file_row = _current_row_group_first_row + _current_row_group_rows_read; @@ -1896,6 +2048,10 @@ Status ParquetScanScheduler::read_next_batch( _current_range_rows_read = 0; } if (*rows == 0) { + // Output-width feedback has no sample for a fully rejected batch. Grow only the + // predicate-side physical read so a 32-row probe cannot pin a long filtered prefix; + // any eventual output still stays below RuntimeState's row cap. + predicate_batch_rows = grow_empty_predicate_batch(predicate_batch_rows); continue; } *eof = false; diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index b885dc6ec329d7..2fcdf253b328bd 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -104,14 +104,18 @@ struct RowGroupReadPlan { std::vector selected_ranges; // row ranges to read after page-index pruning std::map page_skip_plans; // leaf_column_id -> data pages that can be skipped completely - // Native planning already parsed these indexes. Transfer them to execution so narrowed scans - // do not issue the same remote index reads a second time while opening the row group. + // Deferred planning transfers parsed indexes to execution so narrowed scans never issue the + // same remote index reads a second time while opening the row group. std::unordered_map offset_indexes; + // Footer statistics are cheap and eager. Remote dictionary/Bloom/page-index probes fill the + // remaining fields only when this row group reaches the scheduler. + bool expensive_pruning_pending = false; }; struct RowGroupScanPlan { std::vector row_groups; // row groups selected after pruning ParquetPruningStats pruning_stats; // pruning statistics + bool enable_bloom_filter = false; }; // ============================================================================ @@ -126,6 +130,14 @@ Status plan_parquet_row_groups(const NativeParquetMetadata& metadata, ParquetFileContext* file_context = nullptr, const ParquetColumnReaderProfile& column_reader_profile = {}); +Status finalize_parquet_row_group_plans( + const NativeParquetMetadata& metadata, + const std::vector>& file_schema, + const format::FileScanRequest& request, bool enable_bloom_filter, RowGroupScanPlan* plan, + const cctz::time_zone* timezone, const RuntimeState* runtime_state, + ParquetFileContext* file_context, const ParquetColumnReaderProfile& column_reader_profile, + const ParquetProfile* parquet_profile = nullptr); + IColumn::Filter selection_to_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows); @@ -150,6 +162,9 @@ class ParquetScanScheduler { _page_skip_profile = page_skip_profile; } void set_scan_profile(ParquetScanProfile scan_profile) { _scan_profile = scan_profile; } + void set_pruning_profile(const ParquetProfile* parquet_profile) { + _parquet_profile = parquet_profile; + } void set_merge_read_options(RuntimeProfile* profile, int64_t merge_read_slice_size) { _profile = profile; _merge_read_slice_size = merge_read_slice_size; @@ -257,11 +272,13 @@ class ParquetScanScheduler { bool _current_merge_range_active = false; ParquetPageSkipProfile _page_skip_profile; ParquetScanProfile _scan_profile; + const ParquetProfile* _parquet_profile = nullptr; RuntimeProfile* _profile = nullptr; int64_t _merge_read_slice_size = -1; std::optional _global_rowid_context; const cctz::time_zone* _timezone = nullptr; bool _enable_strict_mode = false; + bool _enable_bloom_filter = false; RuntimeState* _runtime_state = nullptr; int64_t _batch_size = DEFAULT_READ_BATCH_SIZE; // Batch control scratch is scheduler-owned so adaptive row caps change logical sizes without diff --git a/be/src/format_v2/parquet/parquet_statistics.cpp b/be/src/format_v2/parquet/parquet_statistics.cpp index 1491cc793313e5..eac5bed00ed531 100644 --- a/be/src/format_v2/parquet/parquet_statistics.cpp +++ b/be/src/format_v2/parquet/parquet_statistics.cpp @@ -80,21 +80,41 @@ Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, return Status::OK(); } -bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, - const tparquet::Statistics& statistics) { +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id) { + return leaf_column_id >= 0 && metadata.__isset.column_orders && + leaf_column_id < static_cast(metadata.column_orders.size()) && + metadata.column_orders[leaf_column_id].__isset.TYPE_ORDER; +} + +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + auto sanitized = statistics; + if (!has_type_defined_order || !sanitized.__isset.min_value || !sanitized.__isset.max_value) { + sanitized.__isset.min_value = false; + sanitized.__isset.max_value = false; + sanitized.min_value.clear(); + sanitized.max_value.clear(); + } const bool binary = type_descriptor.physical_type == tparquet::Type::BYTE_ARRAY || type_descriptor.physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY; - if (!binary) { - return true; - } - if (statistics.__isset.min_value || statistics.__isset.max_value) { - // Do not combine a type-defined bound with a deprecated bound: the two fields can use - // different byte ordering, so a mixed pair cannot form one valid interval. - return statistics.__isset.min_value && statistics.__isset.max_value; - } - // Deprecated binary min/max fields were ordered with signed bytes by legacy parquet-mr, while - // Parquet's type-defined order is unsigned. Only an equal pair is independent of that mismatch. - return statistics.__isset.min && statistics.__isset.max && statistics.min == statistics.max; + if (!sanitized.__isset.min || !sanitized.__isset.max || + (binary && sanitized.min != sanitized.max)) { + sanitized.__isset.min = false; + sanitized.__isset.max = false; + sanitized.min.clear(); + sanitized.max.clear(); + } + return sanitized; +} + +bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order) { + const auto sanitized = + sanitize_native_footer_statistics(type_descriptor, statistics, has_type_defined_order); + return (sanitized.__isset.min_value && sanitized.__isset.max_value) || + (sanitized.__isset.min && sanitized.__isset.max); } } // namespace detail @@ -596,7 +616,8 @@ bool native_metadata_predicate_is_type_safe(const ParquetColumnSchema& column_sc return remove_nullable(column_schema.type)->get_primitive_type() != TYPE_VARBINARY; } -bool check_native_statistics(const tparquet::RowGroup& row_group, +bool check_native_statistics(const tparquet::FileMetaData& metadata, + const tparquet::RowGroup& row_group, const std::vector>& file_schema, const format::FileScanRequest& request, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone) { @@ -620,15 +641,18 @@ bool check_native_statistics(const tparquet::RowGroup& row_group, std::shared_ptr zone_map; if (chunk.__isset.meta_data) { const auto& column_metadata = chunk.meta_data; - const auto* statistics = - column_metadata.__isset.statistics ? &column_metadata.statistics : nullptr; - if (statistics != nullptr && !detail::can_use_native_footer_min_max( - column_schema->type_descriptor, *statistics)) { - statistics = nullptr; + std::optional safe_statistics; + if (column_metadata.__isset.statistics) { + safe_statistics = detail::sanitize_native_footer_statistics( + column_schema->type_descriptor, column_metadata.statistics, + detail::has_supported_type_defined_order(metadata, + column_schema->leaf_column_id)); } zone_map = ParquetStatisticsUtils::MakeZoneMap( ParquetStatisticsUtils::TransformColumnStatistics( - *column_schema, statistics, column_metadata.num_values, timezone)); + *column_schema, + safe_statistics.has_value() ? &*safe_statistics : nullptr, + column_metadata.num_values, timezone)); } add_slot_zonemap(&ctx, slot_index, column_schema->type, std::move(zone_map)); } @@ -847,7 +871,8 @@ Status select_row_groups_by_metadata( std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone, const RuntimeState* runtime_state, ParquetFileContext* file_context, - const ParquetColumnReaderProfile& column_reader_profile) { + const ParquetColumnReaderProfile& column_reader_profile, + ParquetMetadataProbeMode probe_mode) { int64_t timer_sink = 0; SCOPED_RAW_TIMER(pruning_stats == nullptr ? &timer_sink : &pruning_stats->row_group_filter_time); @@ -882,16 +907,20 @@ Status select_row_groups_by_metadata( continue; } ParquetRowGroupPruneReason prune_reason = ParquetRowGroupPruneReason::NONE; - if (has_expr_zonemap_filter(request, runtime_state) && - check_native_statistics(row_group, file_schema, request, pruning_stats, timezone)) { + if (probe_mode != ParquetMetadataProbeMode::EXPENSIVE_ONLY && + has_expr_zonemap_filter(request, runtime_state) && + check_native_statistics(metadata, row_group, file_schema, request, pruning_stats, + timezone)) { prune_reason = ParquetRowGroupPruneReason::STATISTICS; } - if (prune_reason == ParquetRowGroupPruneReason::NONE) { + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE) { prune_reason = native_dictionary_prune_reason(row_group, row_group_idx, file_schema, request, timezone, file_context, column_reader_profile); } - if (prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { + if (probe_mode != ParquetMetadataProbeMode::FOOTER_ONLY && + prune_reason == ParquetRowGroupPruneReason::NONE && enable_bloom_filter) { prune_reason = native_bloom_filter_prune_reason(row_group, file_schema, request, file_context, pruning_stats); } @@ -1142,6 +1171,7 @@ RowRange native_page_row_range(const tparquet::OffsetIndex& offset_index, size_t } // namespace Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, const std::unordered_map& page_indexes, const std::vector>& file_schema, const format::FileScanRequest& request, int64_t row_group_rows, @@ -1179,7 +1209,8 @@ Status select_row_group_ranges_by_native_page_index( } const auto* column_schema = resolve_local_leaf_schema(file_schema, *file_column_id); if (column_schema == nullptr || column_schema->type == nullptr || - !native_metadata_predicate_is_type_safe(*column_schema)) { + !native_metadata_predicate_is_type_safe(*column_schema) || + !detail::has_supported_type_defined_order(metadata, column_schema->leaf_column_id)) { continue; } const auto index_it = page_indexes.find(column_schema->leaf_column_id); diff --git a/be/src/format_v2/parquet/parquet_statistics.h b/be/src/format_v2/parquet/parquet_statistics.h index 47eedc39ce3c52..72381548656f9d 100644 --- a/be/src/format_v2/parquet/parquet_statistics.h +++ b/be/src/format_v2/parquet/parquet_statistics.h @@ -56,7 +56,12 @@ Status validate_native_bloom_filter_layout(int64_t offset, uint32_t header_size, int64_t payload_size, int64_t declared_length, size_t file_size); bool can_use_native_footer_min_max(const ParquetTypeDescriptor& type_descriptor, - const tparquet::Statistics& statistics); + const tparquet::Statistics& statistics, + bool has_type_defined_order); +bool has_supported_type_defined_order(const tparquet::FileMetaData& metadata, int leaf_column_id); +tparquet::Statistics sanitize_native_footer_statistics(const ParquetTypeDescriptor& type_descriptor, + const tparquet::Statistics& statistics, + bool has_type_defined_order); } // namespace detail // ============================================================================ @@ -100,6 +105,12 @@ struct NativeParquetPageIndex { tparquet::OffsetIndex offset_index; }; +enum class ParquetMetadataProbeMode { + ALL, + FOOTER_ONLY, + EXPENSIVE_ONLY, +}; + bool can_use_parquet_page_index(const format::FileScanRequest& request, const RuntimeState* runtime_state); @@ -130,9 +141,11 @@ Status select_row_groups_by_metadata( std::vector* selected_row_groups, bool enable_bloom_filter, ParquetPruningStats* pruning_stats, const cctz::time_zone* timezone = nullptr, const RuntimeState* runtime_state = nullptr, ParquetFileContext* file_context = nullptr, - const ParquetColumnReaderProfile& column_reader_profile = {}); + const ParquetColumnReaderProfile& column_reader_profile = {}, + ParquetMetadataProbeMode probe_mode = ParquetMetadataProbeMode::ALL); Status select_row_group_ranges_by_native_page_index( + const tparquet::FileMetaData& metadata, const std::unordered_map& page_indexes, const std::vector>& file_schema, const format::FileScanRequest& request, int64_t row_group_rows, diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 0b39ba04af67d3..6f104acc8cf488 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -445,6 +445,14 @@ bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { return false; } +bool is_decimal_column(const IColumn& column) { + return check_and_get_column(&column) != nullptr || + check_and_get_column(&column) != nullptr || + check_and_get_column(&column) != nullptr || + check_and_get_column(&column) != nullptr || + check_and_get_column(&column) != nullptr; +} + template void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, const NullMap& selected_nulls) { @@ -1262,11 +1270,12 @@ Status ColumnChunkReader::materialize_values( state, select_vector, &_chunk_statistics.materialization_time); _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); - } else if (context.encoding == ParquetValueEncoding::DICTIONARY && + } else if ((context.encoding == ParquetValueEncoding::DICTIONARY || + is_decimal_column(*doris_column)) && visit_nullable_expandable_column(*doris_column, [](auto&) {})) { - // PLAIN fixed-width pages already skip by pointer arithmetic; compact-and-expand is a - // net loss there. Keep this StarRocks-style nullable batching on dictionary pages, - // where it also collapses thousands of one-value RLE decoder calls into range reads. + // Nullable DECIMAL conversion is substantially heavier than PLAIN cursor arithmetic. + // Keep its SerDe/consumer alive for the whole sparse request so NULLs cannot turn one + // physical batch into thousands of tiny conversion calls. ++_chunk_statistics.hybrid_selection_batches; status = decode_selected_nullable_values( *doris_column, serde, *_page_decoder, context, state, select_vector, diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index 526a6ecfb3590a..830f86e204bf3c 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -28,6 +29,10 @@ #include #include +#ifdef __AVX2__ +#include +#endif + #include "common/status.h" #include "core/custom_allocator.h" #include "core/data_type_serde/parquet_decode_source.h" @@ -36,6 +41,35 @@ #include "util/slice.h" namespace doris::format::parquet::native { + +inline bool dictionary_indices_in_bounds(const uint32_t* indices, size_t count, + size_t dictionary_size) { + if (count == 0) { + return true; + } + if (dictionary_size == 0) { + return false; + } + uint32_t max_index = 0; + size_t row = 0; +#ifdef __AVX2__ + __m256i vector_max = _mm256_setzero_si256(); + for (; row + 8 <= count; row += 8) { + const auto values = _mm256_loadu_si256(reinterpret_cast(indices + row)); + vector_max = _mm256_max_epu32(vector_max, values); + } + alignas(32) uint32_t lanes[8]; + _mm256_store_si256(reinterpret_cast<__m256i*>(lanes), vector_max); + for (const auto lane : lanes) { + max_index = std::max(max_index, lane); + } +#endif + for (; row < count; ++row) { + max_index = std::max(max_index, indices[row]); + } + return static_cast(max_index) < dictionary_size; +} + class Decoder : public ParquetDecodeSource { public: Decoder() = default; @@ -128,8 +162,14 @@ class BaseDictDecoder : public Decoder { return Status::IOError("Can't read enough Parquet dictionary indices"); } const size_t num_dictionary_values = dictionary_size(); - for (size_t row = 0; row < num_values; ++row) { - if (UNLIKELY((*indices)[row] >= num_dictionary_values)) { + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data(), num_values, + num_dictionary_values))) { + // The SIMD common path only computes a bound; recover the exact corrupt row for the + // diagnostic after the batch has already been proven invalid. + for (size_t row = 0; row < num_values; ++row) { + if ((*indices)[row] < num_dictionary_values) { + continue; + } return Status::Corruption( "Parquet dictionary index {} at row {} exceeds dictionary size {}", (*indices)[row], row, num_dictionary_values); @@ -154,8 +194,12 @@ class BaseDictDecoder : public Decoder { if (UNLIKELY(decoded != range.count)) { return Status::IOError("Can't read enough Parquet dictionary indices"); } - for (size_t row = 0; row < range.count; ++row) { - if (UNLIKELY((*indices)[output + row] >= num_dictionary_values)) { + if (UNLIKELY(!dictionary_indices_in_bounds(indices->data() + output, range.count, + num_dictionary_values))) { + for (size_t row = 0; row < range.count; ++row) { + if ((*indices)[output + row] < num_dictionary_values) { + continue; + } return Status::Corruption( "Parquet dictionary index {} at row {} exceeds dictionary size {}", (*indices)[output + row], range.first + row, num_dictionary_values); @@ -200,9 +244,13 @@ class BaseDictDecoder : public Decoder { "Can't skip enough Parquet dictionary indices at row {}: {} of {}", row_offset + skipped_values, skipped, batch_size); } - // Filter gaps may be huge RLE runs; validate them without allocating by gap size. - for (size_t row = 0; row < batch_size; ++row) { - if (UNLIKELY(_skip_indices[row] >= num_dictionary_values)) { + // Filter gaps may be huge RLE runs; validate them in bounded SIMD-sized batches. + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch_size, + num_dictionary_values))) { + for (size_t row = 0; row < batch_size; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } return Status::Corruption( "Parquet dictionary index {} at skipped row {} exceeds dictionary " "size {}", diff --git a/be/test/format_v2/column_mapper_test.cpp b/be/test/format_v2/column_mapper_test.cpp index c0318d451187ed..c5d837bdff756e 100644 --- a/be/test/format_v2/column_mapper_test.cpp +++ b/be/test/format_v2/column_mapper_test.cpp @@ -2410,6 +2410,7 @@ TEST(ColumnMapperScanRequestTest, HiddenTopLevelFilterMappingUsesNameFallback) { EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(1)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(1)})); ASSERT_TRUE(mapper.filter_entries().at(GlobalIndex(1)).is_local()); EXPECT_EQ(mapper.filter_entries().at(GlobalIndex(1)).local_index(), LocalIndex(1)); } @@ -2935,6 +2936,7 @@ TEST(ColumnMapperScanRequestTest, PredicateOnlyTopLevelColumnUsesHiddenMapping) EXPECT_EQ(request.non_predicate_columns[0].column_id(), LocalColumnId(0)); ASSERT_EQ(request.predicate_columns.size(), 1); EXPECT_EQ(request.predicate_columns[0].column_id(), LocalColumnId(10)); + EXPECT_EQ(request.predicate_only_columns, std::vector({LocalColumnId(10)})); EXPECT_TRUE(request.predicate_columns[0].project_all_children); EXPECT_TRUE(request.predicate_columns[0].children.empty()); diff --git a/be/test/format_v2/orc/orc_file_input_stream_test.cpp b/be/test/format_v2/orc/orc_file_input_stream_test.cpp index 54c89505edc3f5..8d63a4ab21ae65 100644 --- a/be/test/format_v2/orc/orc_file_input_stream_test.cpp +++ b/be/test/format_v2/orc/orc_file_input_stream_test.cpp @@ -329,14 +329,49 @@ TEST(OrcFileInputStreamTest, PublishesClusterProfileExactlyOnce) { input.beforeReadStripe( std::make_unique(200, std::vector {}), selected_columns({}), next_streams); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); } - EXPECT_EQ(profile.get_counter("RequestIO")->value(), 2); - EXPECT_EQ(profile.get_counter("MergedIO")->value(), 1); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 2); + EXPECT_EQ(profile.get_counter("OrcMergedIO")->value(), 1); +} + +TEST(OrcFileInputStreamTest, MergedIoChildrenStayIsolatedInBothInitializationOrders) { + for (const bool generic_first : {false, true}) { + auto reader = std::make_shared(256); + RuntimeProfile profile(generic_first ? "generic_first" : "orc_first"); + auto register_generic = [&] { + ADD_TIMER(&profile, "MergedSmallIO"); + return ADD_CHILD_COUNTER_WITH_LEVEL(&profile, "RequestIO", TUnit::UNIT, "MergedSmallIO", + 1); + }; + RuntimeProfile::Counter* generic_request_io = nullptr; + if (generic_first) { + generic_request_io = register_generic(); + } + { + OrcFileInputStream input("test.orc", reader, nullptr, &profile, + {.once_max_read_bytes = 16, .max_merge_distance_bytes = 0}); + if (!generic_first) { + generic_request_io = register_generic(); + } + StripeStreamMap streams; + input.beforeReadStripe( + std::make_unique( + 100, std::vector {{1, ::orc::StreamKind_DATA, 4}, + {2, ::orc::StreamKind_DATA, 4}}), + selected_columns({1, 2}), streams); + std::array data {}; + find_stream(streams, 1, ::orc::StreamKind_DATA)->read(data.data(), data.size(), 100); + } + ASSERT_NE(generic_request_io, nullptr); + EXPECT_EQ(generic_request_io->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + EXPECT_EQ(profile.get_counter("OrcMergedRequestIO")->value(), 1); + } } } // namespace diff --git a/be/test/format_v2/orc/orc_reader_test.cpp b/be/test/format_v2/orc/orc_reader_test.cpp index d231d9fd091dae..20696bcc2ff64a 100644 --- a/be/test/format_v2/orc/orc_reader_test.cpp +++ b/be/test/format_v2/orc/orc_reader_test.cpp @@ -5263,16 +5263,17 @@ TEST_F(NewOrcReaderTest, StripePrefetchPublishesMergedReadProfile) { } ASSERT_TRUE(reader->close().ok()); - ASSERT_NE(profile.get_counter("RequestIO"), nullptr); - ASSERT_NE(profile.get_counter("MergedIO"), nullptr); - ASSERT_NE(profile.get_counter("ApplyBytes"), nullptr); - ASSERT_NE(profile.get_counter("ClusterNum"), nullptr); - ASSERT_NE(profile.get_counter("OverReadBytes"), nullptr); - EXPECT_GT(profile.get_counter("RequestIO")->value(), profile.get_counter("MergedIO")->value()); - EXPECT_GT(profile.get_counter("MergedIO")->value(), 0); - EXPECT_GT(profile.get_counter("ApplyBytes")->value(), 0); - EXPECT_GT(profile.get_counter("ClusterNum")->value(), 0); - EXPECT_GE(profile.get_counter("OverReadBytes")->value(), 0); + ASSERT_NE(profile.get_counter("OrcMergedRequestIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedIO"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedApplyBytes"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedClusterNum"), nullptr); + ASSERT_NE(profile.get_counter("OrcMergedOverReadBytes"), nullptr); + EXPECT_GT(profile.get_counter("OrcMergedRequestIO")->value(), + profile.get_counter("OrcMergedIO")->value()); + EXPECT_GT(profile.get_counter("OrcMergedIO")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedApplyBytes")->value(), 0); + EXPECT_GT(profile.get_counter("OrcMergedClusterNum")->value(), 0); + EXPECT_GE(profile.get_counter("OrcMergedOverReadBytes")->value(), 0); } TEST_F(NewOrcReaderTest, StripePrefetchCanBeDisabledByZeroOnceMaxReadBytes) { diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index f32ad23545398d..9134fbf56992f3 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -29,6 +29,7 @@ #include "core/custom_allocator.h" #include "core/data_type/data_type_date_or_datetime_v2.h" +#include "core/data_type/data_type_decimal.h" #include "core/data_type/data_type_map.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" @@ -372,7 +373,8 @@ Status materialize_selected_plain_int32(const std::vector& physical_val const std::vector& filter_values, const DataType& type, MutableColumnPtr* column, NullMap* null_map, ColumnChunkReaderStatistics* statistics, - bool strict_mode = false) { + bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { tparquet::PageHeader header; header.type = tparquet::PageType::DATA_PAGE; header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); @@ -405,6 +407,9 @@ Status materialize_selected_plain_int32(const std::vector& physical_val ParquetDecodeContext decode_context; decode_context.physical_type = ParquetPhysicalType::INT32; + if (context_override != nullptr) { + decode_context = *context_override; + } ParquetMaterializationState state; state.enable_strict_mode = strict_mode; state.conversion_failure_null_map = null_map; @@ -602,6 +607,42 @@ TEST(ParquetV2NativeDecoderTest, NullableSparsePlainSelectionRetainsCheaperRange EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); } +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysicalPayload) { + constexpr size_t LOGICAL_VALUES = 4095; + std::vector null_runs(LOGICAL_VALUES, 1); + std::vector physical_values((LOGICAL_VALUES + 1) / 2); + std::iota(physical_values.begin(), physical_values.end(), 0); + std::vector filter(LOGICAL_VALUES, 0); + for (const size_t selected_row : {0, 1000, 1001, 2000, 4001}) { + filter[selected_row] = 1; + } + + DataTypeDecimal32 type(9, 0); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DECIMAL, + .decimal_precision = 9, + .decimal_scale = 0}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_values, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + const auto& values = assert_cast(*column).get_data(); + ASSERT_EQ(values.size(), 5); + EXPECT_EQ(values[0].value, 0); + EXPECT_EQ(values[1].value, 500); + EXPECT_EQ(values[2].value, 0); + EXPECT_EQ(values[3].value, 1000); + EXPECT_EQ(values[4].value, 0); + EXPECT_EQ(null_map, (NullMap {0, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { constexpr size_t LOGICAL_VALUES = 4095; std::vector dictionary(16); @@ -643,6 +684,34 @@ TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysica EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); } +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationSkipsFailureScanWhenDictionaryIsClean) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + auto& dictionary = assert_cast(*state.typed_dictionary).get_data(); + dictionary = {10, 20, 30, 40}; + state.dictionary_indices = {3, 0, 2, 1, 3}; + state.dictionary_conversion_failures.resize_fill(dictionary.size(), 0); + + auto output = ColumnInt32::create(); + ASSERT_TRUE(state.materialize_dictionary(*output).ok()); + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_failure_scan_rows, 0); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryIndexBoundsCheckHandlesVectorTailAndUnsignedIds) { + const std::vector valid {0, 7, 1, 6, 2, 5, 3, 4, 7, 0, 6}; + EXPECT_TRUE(native::dictionary_indices_in_bounds(valid.data(), valid.size(), 8)); + + auto invalid_tail = valid; + invalid_tail.back() = 8; + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_tail.data(), invalid_tail.size(), 8)); + + auto invalid_unsigned = valid; + invalid_unsigned[3] = std::numeric_limits::max(); + EXPECT_FALSE(native::dictionary_indices_in_bounds(invalid_unsigned.data(), + invalid_unsigned.size(), 8)); +} + TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionRemapsConversionFailures) { const std::vector dictionary {1, 1000, 2, 3}; const std::vector physical_ids {0, 1, 2, 3}; diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index bc6a2b1948e291..ada2c17580a978 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -2545,7 +2545,15 @@ TEST_F(NewParquetReaderTest, DictionaryPruningPublishesColdAndWarmNativePageProf request->non_predicate_columns = {field_projection(0)}; request->conjuncts.push_back(create_string_in_conjunct(1, {"not-present"})); use_schema_order_positions(request.get(), schema); - return reader->open(request); + RETURN_IF_ERROR(reader->open(request)); + EXPECT_EQ(profile->get_counter("RowGroupsFilteredByDictionary")->value(), 0); + EXPECT_EQ(profile->get_counter("PageReadCount")->value(), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + // Expensive dictionary probes are current-row-group work now, so advance the scheduler + // once before inspecting page-cache and pruning counters. + return reader->get_block(&block, &rows, &eof); }; RuntimeProfile cold_profile("dictionary_pruning_cold_profile"); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index b5ef381dbe1b10..f047337e97e7e9 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -468,6 +468,14 @@ void write_page_index_parquet_file(const std::string& file_path) { write_table(file_path, table, ids.size(), false, true); } +void write_multi_row_group_page_index_parquet_file(const std::string& file_path) { + std::vector ids(384); + std::iota(ids.begin(), ids.end(), 0); + auto schema = arrow::schema({arrow::field("id", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_int32_array(ids)}); + write_table(file_path, table, 128, false, true); +} + int64_t parquet_column_start_offset(const ::parquet::ColumnChunkMetaData& column_metadata) { return column_metadata.has_dictionary_page() ? static_cast(column_metadata.dictionary_page_offset()) @@ -1094,6 +1102,35 @@ TEST_F(ParquetScanTest, PredicateColumnsSkipUnreadColumnsWhenFirstPredicateFilte EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 6); } +TEST_F(ParquetScanTest, PredicateOnlyColumnDropsPayloadAfterFiltering) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + // Scenario: every physical batch in every row group is rejected. Predicate readers reach each row // group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that // reader and its pending lag instead of issuing SkipRecords for values that can never be observed. @@ -1124,7 +1161,9 @@ TEST_F(ParquetScanTest, FullyFilteredRowGroupsDropPendingLazyReaders) { } EXPECT_EQ(total_rows, 0); - EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 6); + // The first one-row probe grows after rejection and remains grown across consecutive empty + // row groups, so later two-row groups are consumed in one predicate batch each. + EXPECT_EQ(counter_value(profile, "EmptySelectionBatches"), 4); EXPECT_EQ(counter_value(profile, "ReaderSkipRows"), 0); ASSERT_NE(profile.get_counter("LevelOnlySkipTime"), nullptr); EXPECT_EQ(profile.get_counter("LevelOnlySkipTime")->value(), 0); @@ -1172,8 +1211,9 @@ TEST_F(ParquetScanTest, PendingLazySkipDoesNotCrossRowGroupReset) { TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunks) { constexpr size_t ROWS = 70000; write_long_prefix_parquet_file(_file_path, ROWS); - auto reader = create_reader(); - reader->set_batch_size(1024); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + reader->set_batch_size(32); RuntimeState state {TQueryOptions(), TQueryGlobals()}; ASSERT_TRUE(reader->init(&state).ok()); @@ -1196,6 +1236,7 @@ TEST_F(ParquetScanTest, LongFilteredPrefixSkipsMultipleLazyColumnsInBoundedChunk EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_element(0), ROWS - 1); EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_element(0), ROWS + 9); EXPECT_EQ(int32_data_column(*block.get_by_position(2).column).get_element(0), ROWS + 19); + EXPECT_LT(counter_value(profile, "TotalBatches"), 32); } // Scenario: a nested lazy column stays behind while id=1 is rejected. Flushing skip(1) must consume @@ -1332,5 +1373,30 @@ TEST_F(ParquetScanTest, ProfileCountersReflectPageIndexAndRangeGapPruning) { EXPECT_GT(profile.get_counter("RangeGapSkippedRows")->value(), 0); } +TEST_F(ParquetScanTest, OpenDefersPageIndexProbeToCurrentRowGroup) { + write_multi_row_group_page_index_parquet_file(_file_path); + RuntimeProfile profile("lazy_page_index_profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->conjuncts.push_back(create_int32_zonemap_conjunct(0, Int32ZoneMapExpr::Op::GT, -1)); + ASSERT_TRUE(reader->open(request).ok()); + + // LIMIT can stop after the first block, so open must not pay remote index I/O for later groups. + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 0); + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + EXPECT_EQ(rows, 128); + EXPECT_EQ(counter_value(profile, "PageIndexReadCalls"), 1); +} + } // namespace } // namespace doris diff --git a/be/test/format_v2/parquet/parquet_statistics_test.cpp b/be/test/format_v2/parquet/parquet_statistics_test.cpp index df486cbc0f3360..7ba9a3cde19746 100644 --- a/be/test/format_v2/parquet/parquet_statistics_test.cpp +++ b/be/test/format_v2/parquet/parquet_statistics_test.cpp @@ -22,10 +22,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -127,6 +129,36 @@ class DictionaryStringInExpr final : public VExpr { private: const std::string _expr_name = "DictionaryStringInExpr"; }; + +class MetadataInt32GreaterThanExpr final : public VExpr { +public: + explicit MetadataInt32GreaterThanExpr(int32_t value) + : VExpr(std::make_shared(), false), _value(value) {} + + const std::string& expr_name() const override { return _expr_name; } + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t, + ColumnPtr&) const override { + return Status::InternalError("MetadataInt32GreaterThanExpr is metadata-only"); + } + bool can_evaluate_zonemap_filter() const override { return true; } + void collect_slot_column_ids(std::set& column_ids) const override { column_ids.insert(0); } + ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override { + const auto zone_map = ctx.zone_map(0); + if (zone_map == nullptr) { + return unsupported_zonemap_filter(ctx); + } + if (!zone_map->has_not_null) { + return ZoneMapFilterResult::kNoMatch; + } + return zone_map->max_value <= Field::create_field(_value) + ? ZoneMapFilterResult::kNoMatch + : ZoneMapFilterResult::kMayMatch; + } + +private: + int32_t _value; + const std::string _expr_name = "MetadataInt32GreaterThanExpr"; +}; VExprContextSPtrs bloom_conjuncts(DataTypePtr data_type, std::vector values) { return {VExprContext::create_shared( std::make_shared(0, std::move(data_type), std::move(values)))}; @@ -321,28 +353,113 @@ TEST(NativeParquetStatisticsTest, LegacyBinaryFooterBoundsRequireComparableOrder tparquet::Statistics max_only; max_only.__set_max("III"); - EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only)); + EXPECT_FALSE( + format::parquet::detail::can_use_native_footer_min_max(binary_type, max_only, false)); tparquet::Statistics legacy_different; legacy_different.__set_min("III"); legacy_different.__set_max("\xe6\x98\xaf"); - EXPECT_FALSE( - format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_different)); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, + legacy_different, false)); tparquet::Statistics legacy_equal; legacy_equal.__set_min("same"); legacy_equal.__set_max("same"); - EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal)); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, legacy_equal, + false)); tparquet::Statistics type_defined; type_defined.__set_min_value("III"); type_defined.__set_max_value("\xe6\x98\xaf"); - EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined)); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + false)); + EXPECT_TRUE(format::parquet::detail::can_use_native_footer_min_max(binary_type, type_defined, + true)); tparquet::Statistics mixed_fields; mixed_fields.__set_min_value("III"); mixed_fields.__set_max("\xe6\x98\xaf"); - EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields)); + EXPECT_FALSE(format::parquet::detail::can_use_native_footer_min_max(binary_type, mixed_fields, + true)); +} + +TEST(NativeParquetStatisticsTest, TypeDefinedBoundsRequireSupportedColumnOrder) { + auto encode_int32 = [](int32_t value) { + std::string bytes(sizeof(value), '\0'); + memcpy(bytes.data(), &value, sizeof(value)); + return bytes; + }; + + auto column_schema = std::make_unique(); + column_schema->kind = format::parquet::ParquetColumnSchemaKind::PRIMITIVE; + column_schema->local_id = 0; + column_schema->leaf_column_id = 0; + column_schema->type = std::make_shared(); + column_schema->type_descriptor.doris_type = column_schema->type; + column_schema->type_descriptor.physical_type = tparquet::Type::INT32; + std::vector> schema; + schema.push_back(std::move(column_schema)); + + tparquet::Statistics statistics; + statistics.__set_min_value(encode_int32(1)); + statistics.__set_max_value(encode_int32(2)); + statistics.__set_null_count(0); + tparquet::ColumnMetaData column_metadata; + column_metadata.__set_type(tparquet::Type::INT32); + column_metadata.__set_num_values(1); + column_metadata.__set_statistics(statistics); + tparquet::ColumnChunk chunk; + chunk.__set_meta_data(column_metadata); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_num_rows(1); + tparquet::FileMetaData metadata; + metadata.__set_row_groups({row_group}); + + format::FileScanRequest request; + request.local_positions.emplace(format::LocalColumnId(0), format::LocalIndex(0)); + request.predicate_columns = {format::LocalColumnIndex::top_level(format::LocalColumnId(0))}; + request.conjuncts = { + VExprContext::create_shared(std::make_shared(100))}; + std::vector selected_row_groups; + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) + .ok()); + EXPECT_EQ(selected_row_groups, std::vector({0})); + + format::parquet::NativeParquetPageIndex page_index; + page_index.column_index.__set_min_values({encode_int32(1)}); + page_index.column_index.__set_max_values({encode_int32(2)}); + page_index.column_index.__set_null_pages({false}); + page_index.column_index.__set_null_counts({0}); + tparquet::PageLocation location; + location.__set_offset(0); + location.__set_compressed_page_size(10); + location.__set_first_row_index(0); + page_index.offset_index.__set_page_locations({location}); + std::unordered_map page_indexes; + page_indexes.emplace(0, page_index); + std::vector selected_ranges; + std::map skip_plans; + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) + .ok()); + EXPECT_EQ(selected_ranges.size(), 1); + + tparquet::ColumnOrder order; + order.__set_TYPE_ORDER(tparquet::TypeDefinedOrder()); + metadata.__set_column_orders({order}); + selected_row_groups.clear(); + ASSERT_TRUE(format::parquet::select_row_groups_by_metadata(metadata, schema, request, nullptr, + &selected_row_groups, false, nullptr) + .ok()); + EXPECT_TRUE(selected_row_groups.empty()); + ASSERT_TRUE(format::parquet::select_row_group_ranges_by_native_page_index( + metadata, page_indexes, schema, request, 1, &selected_ranges, &skip_plans, + nullptr) + .ok()); + EXPECT_TRUE(selected_ranges.empty()); } } // namespace diff --git a/be/test/format_v2/table_reader_test.cpp b/be/test/format_v2/table_reader_test.cpp index 3a6cf401019337..3e05a8dbf91264 100644 --- a/be/test/format_v2/table_reader_test.cpp +++ b/be/test/format_v2/table_reader_test.cpp @@ -4068,6 +4068,7 @@ TEST(TableReaderTest, CreateScanRequestPromotesProjectedColumnToPredicateColumn) EXPECT_EQ(projection_ids(file_request.predicate_columns), std::vector({0})); EXPECT_EQ(projection_ids(file_request.non_predicate_columns), std::vector({1})); + EXPECT_TRUE(file_request.predicate_only_columns.empty()); ASSERT_EQ(file_request.local_positions.size(), 2); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(0)).value(), 1); EXPECT_EQ(file_request.local_positions.at(LocalColumnId(1)).value(), 0); From 924c4df3613b0f8417d5670425035d3ec86d9466 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 14:22:25 +0800 Subject: [PATCH 29/34] [opt](be) batch nullable temporal sparse decode --- .../reader/native/column_chunk_reader.cpp | 25 +++- .../format_v2/parquet/native_decoder_test.cpp | 112 +++++++++++++++--- 2 files changed, 116 insertions(+), 21 deletions(-) diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 6f104acc8cf488..d8ae39b27f1d6e 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -445,12 +445,14 @@ bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { return false; } -bool is_decimal_column(const IColumn& column) { +bool has_expensive_nullable_sparse_materialization(const IColumn& column) { return check_and_get_column(&column) != nullptr || check_and_get_column(&column) != nullptr || check_and_get_column(&column) != nullptr || check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr; + check_and_get_column(&column) != nullptr || + check_and_get_column(&column) != nullptr || + check_and_get_column(&column) != nullptr; } template @@ -1271,11 +1273,22 @@ Status ColumnChunkReader::materialize_values( &_chunk_statistics.materialization_time); _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); } else if ((context.encoding == ParquetValueEncoding::DICTIONARY || - is_decimal_column(*doris_column)) && + has_expensive_nullable_sparse_materialization(*doris_column)) && visit_nullable_expandable_column(*doris_column, [](auto&) {})) { - // Nullable DECIMAL conversion is substantially heavier than PLAIN cursor arithmetic. - // Keep its SerDe/consumer alive for the whole sparse request so NULLs cannot turn one - // physical batch into thousands of tiny conversion calls. + // Parquet omits NULL leaf values from the physical stream. For example, the logical + // DATE sequence [d0, NULL, d1, NULL, d2] is physically encoded as [d0, d1, d2]. The + // generic fallback follows the logical selection runs, so those NULLs split one + // contiguous physical span into decode(d0), insert-NULL, decode(d1), insert-NULL, + // decode(d2). On nullable sparse scans this repeatedly enters SerDe and turns decimal + // scaling, date conversion, timestamp/timezone conversion, or dictionary-ID + // materialization into many tiny calls even though the physical values are adjacent. + // + // Build selected non-NULL ranges in physical coordinates instead. In the example this + // decodes [d0, d1, d2] compactly with one SerDe consumer, records [0, 1, 0, 1, 0] as + // the logical NULL layout, and expands the final nested column backwards so unread + // compact values cannot be overwritten. Keep cheap non-decimal PLAIN numeric columns + // on the fallback: their skips are pointer arithmetic, while compact-and-expand would + // add a full output-column memory pass without amortizing expensive conversion work. ++_chunk_statistics.hybrid_selection_batches; status = decode_selected_nullable_values( *doris_column, serde, *_page_decoder, context, state, select_vector, diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 9134fbf56992f3..9b9f7b511118be 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -366,25 +366,24 @@ Status materialize_plain_int96(const std::vector& values, select_vector); } -template -Status materialize_selected_plain_int32(const std::vector& physical_values, - size_t logical_values, - const std::vector& run_length_null_map, - const std::vector& filter_values, - const DataType& type, MutableColumnPtr* column, - NullMap* null_map, ColumnChunkReaderStatistics* statistics, - bool strict_mode = false, - const ParquetDecodeContext* context_override = nullptr) { +template +Status materialize_selected_plain_fixed( + const std::vector& physical_values, size_t logical_values, + const std::vector& run_length_null_map, const std::vector& filter_values, + tparquet::Type::type parquet_physical_type, ParquetPhysicalType decode_physical_type, + const DataType& type, MutableColumnPtr* column, NullMap* null_map, + ColumnChunkReaderStatistics* statistics, bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { tparquet::PageHeader header; header.type = tparquet::PageType::DATA_PAGE; - header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); - header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_compressed_page_size(physical_values.size() * sizeof(PhysicalType)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(PhysicalType)); header.__isset.data_page_header = true; header.data_page_header.__set_num_values(static_cast(logical_values)); header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); - std::vector payload(physical_values.size() * sizeof(int32_t)); + std::vector payload(physical_values.size() * sizeof(PhysicalType)); if (!payload.empty()) { memcpy(payload.data(), physical_values.data(), payload.size()); } @@ -392,13 +391,13 @@ Status materialize_selected_plain_int32(const std::vector& physical_val MemoryBufferedReader reader(bytes); tparquet::ColumnChunk chunk; - chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_type(parquet_physical_type); chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); chunk.meta_data.__set_num_values(static_cast(logical_values)); chunk.meta_data.__set_total_compressed_size(bytes.size()); chunk.meta_data.__set_data_page_offset(0); NativeFieldSchema field; - field.physical_type = tparquet::Type::INT32; + field.physical_type = parquet_physical_type; ParquetPageReadContext page_context(false, ""); ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, logical_values, nullptr, page_context); @@ -406,7 +405,7 @@ Status materialize_selected_plain_int32(const std::vector& physical_val RETURN_IF_ERROR(chunk_reader.load_page_data()); ParquetDecodeContext decode_context; - decode_context.physical_type = ParquetPhysicalType::INT32; + decode_context.physical_type = decode_physical_type; if (context_override != nullptr) { decode_context = *context_override; } @@ -423,6 +422,35 @@ Status materialize_selected_plain_int32(const std::vector& physical_val return Status::OK(); } +template +Status materialize_selected_plain_int32(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + bool strict_mode = false, + const ParquetDecodeContext* context_override = nullptr) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT32, + ParquetPhysicalType::INT32, type, column, null_map, + statistics, strict_mode, context_override); +} + +template +Status materialize_selected_plain_int64(const std::vector& physical_values, + size_t logical_values, + const std::vector& run_length_null_map, + const std::vector& filter_values, + const DataType& type, MutableColumnPtr* column, + NullMap* null_map, ColumnChunkReaderStatistics* statistics, + const ParquetDecodeContext& context) { + return materialize_selected_plain_fixed(physical_values, logical_values, run_length_null_map, + filter_values, tparquet::Type::INT64, + ParquetPhysicalType::INT64, type, column, null_map, + statistics, false, &context); +} + template Status materialize_selected_dictionary_int32( const std::vector& dictionary, const std::vector& physical_ids, @@ -643,6 +671,60 @@ TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysi EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); } +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateSelectionBatchesPhysicalPayload) { + const std::vector physical_days {0, 1, 2, 3, 4}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateV2 type; + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .logical_type = ParquetLogicalType::DATE}; + ASSERT_TRUE(materialize_selected_plain_int32(physical_days, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, false, + &context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-02"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-04"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDateTimeSelectionBatchesPhysicalPayload) { + const std::vector physical_micros {0, 1'000'000, 2'000'000, 3'000'000, 4'000'000}; + constexpr size_t LOGICAL_VALUES = 9; + const std::vector null_runs(LOGICAL_VALUES, 1); + const std::vector filter {1, 1, 1, 0, 0, 1, 1, 1, 0}; + + DataTypeDateTimeV2 type(6); + auto column = type.create_column(); + NullMap null_map; + ColumnChunkReaderStatistics statistics; + const ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT64, + .logical_type = ParquetLogicalType::TIMESTAMP, + .time_unit = ParquetTimeUnit::MICROS}; + ASSERT_TRUE(materialize_selected_plain_int64(physical_micros, LOGICAL_VALUES, null_runs, filter, + type, &column, &null_map, &statistics, context) + .ok()); + + ASSERT_EQ(column->size(), 6); + EXPECT_EQ(type.to_string(*column, 0), "1970-01-01 00:00:00.000000"); + EXPECT_EQ(type.to_string(*column, 2), "1970-01-01 00:00:01.000000"); + EXPECT_EQ(type.to_string(*column, 4), "1970-01-01 00:00:03.000000"); + EXPECT_EQ(null_map, (NullMap {0, 1, 0, 1, 0, 1})); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 2); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); +} + TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysicalPayload) { constexpr size_t LOGICAL_VALUES = 4095; std::vector dictionary(16); From 503e01c5105f1483072fd5cdeae4f967a64fae06 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 16:20:34 +0800 Subject: [PATCH 30/34] [opt](be) complete parquet v2 sparse predicate paths --- .../data_type_datetimev2_serde.cpp | 4 +- .../data_type_datev2_serde.cpp | 4 +- .../data_type_decimal_serde.cpp | 4 +- .../data_type_number_serde.cpp | 6 +- .../data_type_string_serde.cpp | 6 +- .../data_type_serde/data_type_time_serde.cpp | 4 +- .../data_type_timestamptz_serde.cpp | 4 +- .../data_type_varbinary_serde.cpp | 6 +- .../data_type_serde/parquet_decode_source.h | 84 ++++++- .../format_v2/parquet/native_schema_desc.cpp | 45 +++- be/src/format_v2/parquet/native_schema_desc.h | 6 +- be/src/format_v2/parquet/parquet_profile.cpp | 6 + be/src/format_v2/parquet/parquet_profile.h | 4 + be/src/format_v2/parquet/parquet_scan.cpp | 171 ++++++++++++- .../parquet/reader/column_reader.cpp | 11 + .../format_v2/parquet/reader/column_reader.h | 8 + .../reader/native/column_chunk_reader.cpp | 165 ++++++++++-- .../reader/native/column_chunk_reader.h | 9 + .../parquet/reader/native/column_reader.cpp | 144 +++++++++++ .../parquet/reader/native/column_reader.h | 24 ++ .../format_v2/parquet/reader/native/decoder.h | 74 ++++++ .../parquet/reader/native_column_reader.cpp | 82 ++++++ .../parquet/reader/native_column_reader.h | 8 + .../parquet/reader/plain_fixed_predicate.h | 125 ++++++++++ be/src/util/cpu_info.cpp | 11 + be/src/util/cpu_info.h | 4 + .../format_v2/parquet/native_decoder_test.cpp | 236 +++++++++++++++++- .../format_v2/parquet/parquet_scan_test.cpp | 65 +++++ .../format_v2/parquet/parquet_schema_test.cpp | 98 ++++++++ docs/file-scanner-v2-code-review-guide.md | 23 +- docs/file-scanner-v2-parquet-scan-design.md | 42 +++- 31 files changed, 1382 insertions(+), 101 deletions(-) create mode 100644 be/src/format_v2/parquet/reader/plain_fixed_predicate.h diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp index fcf369d4263f38..65153c2d8d0b49 100644 --- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp @@ -705,9 +705,7 @@ Status DataTypeDateTimeV2SerDe::read_column_from_parquet(IColumn& column, DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - return state.materialize_dictionary(column); + return state.materialize_dictionary(column, source, num_values); } Status DataTypeDateTimeV2SerDe::write_column_to_mysql_binary(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp b/be/src/core/data_type_serde/data_type_datev2_serde.cpp index c54bd1dd678823..04b39b87282b52 100644 --- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp +++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp @@ -228,9 +228,7 @@ Status DataTypeDateV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - return state.materialize_dictionary(column); + return state.materialize_dictionary(column, source, num_values); } Status DataTypeDateV2SerDe::write_column_to_mysql_binary(const IColumn& column, diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp b/be/src/core/data_type_serde/data_type_decimal_serde.cpp index 6174507de54360..3193583ec07a00 100644 --- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp +++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp @@ -787,9 +787,7 @@ Status DataTypeDecimalSerDe::read_column_from_parquet(IColumn& column, DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - return state.materialize_dictionary(column); + return state.materialize_dictionary(column, source, num_values); } template diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp b/be/src/core/data_type_serde/data_type_number_serde.cpp index 783884a073f085..44f2cb2cbb1213 100644 --- a/be/src/core/data_type_serde/data_type_number_serde.cpp +++ b/be/src/core/data_type_serde/data_type_number_serde.cpp @@ -224,7 +224,7 @@ Status append_parquet_number(PaddedPODArray& data, const uint8_t* if constexpr (std::is_same_v) { // Identical fixed-width physical/logical types need no validation or conversion. Parquet // PLAIN values and Doris POD columns share the byte representation on supported targets, - // so preserve the dense vector-at-a-time memcpy invariant used by v1 and DuckDB. + // so preserve one dense vector-at-a-time memcpy instead of converting value by value. memcpy(data.data() + old_size, values, num_values * sizeof(SourceType)); return Status::OK(); } @@ -614,9 +614,7 @@ Status DataTypeNumberSerDe::read_column_from_parquet(IColumn& column, DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - return state.materialize_dictionary(column); + return state.materialize_dictionary(column, source, num_values); } } diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp b/be/src/core/data_type_serde/data_type_string_serde.cpp index 01130b0ddf726e..24f7215d42e044 100644 --- a/be/src/core/data_type_serde/data_type_string_serde.cpp +++ b/be/src/core/data_type_serde/data_type_string_serde.cpp @@ -607,11 +607,7 @@ Status DataTypeStringSerDeBase::read_column_from_parquet( DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - return Status::OK(); + return state.materialize_dictionary(column, source, num_values); } template diff --git a/be/src/core/data_type_serde/data_type_time_serde.cpp b/be/src/core/data_type_serde/data_type_time_serde.cpp index 7027b1e3397cca..52d387eda5ca6d 100644 --- a/be/src/core/data_type_serde/data_type_time_serde.cpp +++ b/be/src/core/data_type_serde/data_type_time_serde.cpp @@ -297,9 +297,7 @@ Status DataTypeTimeV2SerDe::read_column_from_parquet(IColumn& column, ParquetDec DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - return state.materialize_dictionary(column); + return state.materialize_dictionary(column, source, num_values); } template diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp index aab8adc4e98988..97c9582ae31095 100644 --- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp @@ -434,9 +434,7 @@ Status DataTypeTimeStampTzSerDe::read_column_from_parquet( DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - return state.materialize_dictionary(column); + return state.materialize_dictionary(column, source, num_values); } std::string DataTypeTimeStampTzSerDe::to_olap_string(const Field& field) const { diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp index 9bdd5abfcc2bfe..14d5edda0afe1a 100644 --- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp +++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp @@ -87,11 +87,7 @@ Status DataTypeVarbinarySerDe::read_column_from_parquet(IColumn& column, DORIS_CHECK_EQ(state.typed_dictionary->size(), source.dictionary_size()); state.dictionary_generation = source.dictionary_generation(); } - RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &state.dictionary_indices)); - DORIS_CHECK_EQ(state.dictionary_indices.size(), num_values); - column.insert_indices_from(*state.typed_dictionary, state.dictionary_indices.data(), - state.dictionary_indices.data() + num_values); - return Status::OK(); + return state.materialize_dictionary(column, source, num_values); } void DataTypeVarbinarySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index 7055051dc19923..aafb112de946f4 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -140,10 +140,24 @@ class ParquetBinaryValueConsumer { } }; -// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs are -// intentionally excluded: the native ColumnReader uses this plan only when the batch has no NULL -// leaf slots, so selected values can be appended in one pass without a temporary nullable column. -// Ranges are sorted, disjoint, and expressed in the physical value stream's coordinate space. +// Dictionary decoders publish validated IDs without knowing the destination Doris type. A +// cache-resident dictionary can therefore fuse RLE decode with target-column gathering, while a +// large sparse dictionary can still choose a compact ID buffer before random dictionary access. +class ParquetDictionaryValueConsumer { +public: + virtual ~ParquetDictionaryValueConsumer() = default; + virtual Status consume_indices(const uint32_t* indices, size_t num_values) = 0; + virtual Status consume_repeated(uint32_t index, size_t num_values) { + std::vector indices(num_values, index); + return consume_indices(indices.data(), indices.size()); + } +}; + +// Physical value ranges selected from one page-bounded decode request. Definition-level NULLs do +// not occupy the encoded value stream, so the native ColumnReader merges the logical selection +// with definition levels before constructing this plan. The SerDe sees only selected non-NULL +// payload, while the reader restores selected NULL slots after the compact decode. Ranges are +// sorted, disjoint, and expressed in the physical value stream's coordinate space. struct ParquetSelection { size_t total_values = 0; size_t selected_values = 0; @@ -220,8 +234,25 @@ class ParquetDecodeSource { DORIS_CHECK_EQ(indices->size(), selection.selected_values); return Status::OK(); } + virtual Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_dictionary_indices(num_values, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual Status decode_selected_dictionary_values( + const ParquetSelection& selection, ParquetDictionaryValueConsumer& consumer) { + std::vector indices; + RETURN_IF_ERROR(decode_selected_dictionary_indices(selection, &indices)); + return consumer.consume_indices(indices.data(), indices.size()); + } + virtual bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const { + return false; + } }; +enum class ParquetDictionaryMaterializationStrategy : uint8_t { DIRECT, INDICES }; + // Dictionary values are materialized once into the selected Doris type. The state belongs to a // column reader rather than DataTypeSerDe because a SerDe instance can be shared by many files. struct ParquetMaterializationState { @@ -235,6 +266,8 @@ struct ParquetMaterializationState { bool capturing_dictionary_conversion_failures = false; bool dictionary_has_conversion_failures = false; size_t dictionary_failure_scan_rows = 0; + ParquetDictionaryMaterializationStrategy dictionary_materialization_strategy = + ParquetDictionaryMaterializationStrategy::INDICES; void reset_dictionary() { typed_dictionary.reset(); @@ -313,6 +346,49 @@ struct ParquetMaterializationState { } return Status::OK(); } + + Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, + size_t num_values) { + DORIS_CHECK(typed_dictionary); + if (UNLIKELY(dictionary_has_conversion_failures) || + source.prefer_dictionary_index_materialization(typed_dictionary->byte_size())) { + dictionary_materialization_strategy = + ParquetDictionaryMaterializationStrategy::INDICES; + RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &dictionary_indices)); + DORIS_CHECK_EQ(dictionary_indices.size(), num_values); + return materialize_dictionary(column); + } + + class ColumnConsumer final : public ParquetDictionaryValueConsumer { + public: + ColumnConsumer(IColumn& destination, const IColumn& dictionary) + : _destination(destination), _dictionary(dictionary) {} + + Status consume_indices(const uint32_t* indices, size_t num_values) override { + _destination.insert_indices_from(_dictionary, indices, indices + num_values); + return Status::OK(); + } + + Status consume_repeated(uint32_t index, size_t num_values) override { + _destination.insert_many_from(_dictionary, index, num_values); + return Status::OK(); + } + + private: + IColumn& _destination; + const IColumn& _dictionary; + } consumer(column, *typed_dictionary); + + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::DIRECT; + const size_t old_size = column.size(); + const Status status = source.decode_dictionary_values(num_values, consumer); + if (!status.ok()) { + // Streaming direct gather may discover a corrupt late ID after earlier valid runs; + // restore the same all-or-nothing column invariant as the index-buffer path. + column.resize(old_size); + } + return status; + } }; } // namespace doris diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index 7ce1673a0ae6b4..727b8308ec4b96 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -176,12 +176,15 @@ static void set_child_node_level(NativeFieldSchema* parent, int16_t repeated_par } } -static bool is_struct_list_node(const tparquet::SchemaElement& schema) { - const std::string& name = schema.name; - static const Slice array_slice("array", 5); - static const Slice tuple_slice("_tuple", 6); - Slice slice(name); - return slice == array_slice || slice.ends_with(tuple_slice); +static bool is_struct_list_node(const tparquet::SchemaElement& schema, + const std::string& enclosing_list_name) { + // The legacy Parquet exception is exact: accepting every "*_tuple" wrapper changes a standard + // one-child LIST wrapper from ARRAY to ARRAY>. + return schema.name == "array" || schema.name == enclosing_list_name + "_tuple"; +} + +static bool has_logical_annotation(const tparquet::SchemaElement& schema) { + return schema.__isset.logicalType || schema.__isset.converted_type; } std::string NativeFieldSchema::debug_string() const { @@ -536,7 +539,7 @@ Status NativeFieldDescriptor::parse_group_field( Status NativeFieldDescriptor::parse_list_field( const std::vector& t_schemas, size_t curr_pos, - NativeFieldSchema* list_field) { + NativeFieldSchema* list_field, bool repeated_node_is_enclosing_list_element) { // the list definition: // spark and hive have three level schemas but with different schema name // spark: - "list" - "element" @@ -552,7 +555,8 @@ Status NativeFieldDescriptor::parse_list_field( return Status::InvalidArgument("List element should have the second level schema"); } - if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED) { + if (first_level.repetition_type == tparquet::FieldRepetitionType::REPEATED && + !repeated_node_is_enclosing_list_element) { return Status::InvalidArgument("List element can't be a repeated schema"); } @@ -574,7 +578,22 @@ Status NativeFieldDescriptor::parse_list_field( size_t num_children = num_children_node(second_level); if (num_children > 0) { - if (num_children == 1 && !is_struct_list_node(second_level)) { + const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); + const auto& only_child = t_schemas[curr_pos + 2]; + if (num_children == 1 && !structural_wrapper && + has_logical_annotation(second_level)) { + // The repeated node is already the outer LIST element. Preserve its own LIST/MAP + // annotation, but do not interpret its REPEATED marker as another outer array. + set_child_node_level(list_field, list_field->definition_level); + if (is_list_node(second_level)) { + RETURN_IF_ERROR(parse_list_field(t_schemas, curr_pos + 1, list_child, true)); + } else if (is_map_node(second_level)) { + RETURN_IF_ERROR(parse_map_field(t_schemas, curr_pos + 1, list_child, true)); + } else { + RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); + } + } else if (num_children == 1 && !structural_wrapper && + !is_repeated_node(only_child)) { // optional field, and the third level element is the nested structure in list // produce nested structure like: LIST, LIST, LIST> // skip bag/list, it's a repeated element. @@ -606,7 +625,8 @@ Status NativeFieldDescriptor::parse_list_field( } Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, - size_t curr_pos, NativeFieldSchema* map_field) { + size_t curr_pos, NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element) { // the map definition in parquet: // optional group (MAP) { // repeated group map (MAP_KEY_VALUE) { @@ -623,7 +643,7 @@ Status NativeFieldDescriptor::parse_map_field(const std::vector& t_schemas, size_t curr_pos, - NativeFieldSchema* list_field); + NativeFieldSchema* list_field, + bool repeated_node_is_enclosing_list_element = false); Status parse_map_field(const std::vector& t_schemas, size_t curr_pos, - NativeFieldSchema* map_field); + NativeFieldSchema* map_field, + bool repeated_node_is_enclosing_list_element = false); Status parse_struct_field(const std::vector& t_schemas, size_t curr_pos, NativeFieldSchema* struct_field); diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index 884a264cefd2cf..c5608bcaaa8f08 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -175,6 +175,10 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::BYTES, parquet_profile, 1); predicate_compaction_count = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PredicateCompactionCount", TUnit::UNIT, parquet_profile, 1); + plain_predicate_direct_batches = ADD_CHILD_COUNTER_WITH_LEVEL( + profile, "PlainPredicateDirectBatches", TUnit::UNIT, parquet_profile, 1); + plain_predicate_direct_rows = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "PlainPredicateDirectRows", + TUnit::UNIT, parquet_profile, 1); dict_filter_rewrite_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "DictFilterRewriteTime", parquet_profile, 1); dict_filter_expr_rewrite_time = @@ -314,6 +318,8 @@ ParquetScanProfile ParquetProfile::scan_profile() const { .predicate_compaction_time = predicate_compaction_time, .predicate_compaction_bytes = predicate_compaction_bytes, .predicate_compaction_count = predicate_compaction_count, + .plain_predicate_direct_batches = plain_predicate_direct_batches, + .plain_predicate_direct_rows = plain_predicate_direct_rows, .dict_filter_rewrite_time = dict_filter_rewrite_time, .dict_filter_expr_rewrite_time = dict_filter_expr_rewrite_time, .dict_filter_read_dict_time = dict_filter_read_dict_time, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index a225a352296c4b..60289208188922 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -89,6 +89,8 @@ struct ParquetScanProfile { RuntimeProfile::Counter* predicate_compaction_time = nullptr; RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; // dictionary rewrite time (ns) RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; // expression/residual rewrite time (ns) @@ -202,6 +204,8 @@ struct ParquetProfile { RuntimeProfile::Counter* predicate_compaction_time = nullptr; RuntimeProfile::Counter* predicate_compaction_bytes = nullptr; RuntimeProfile::Counter* predicate_compaction_count = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_batches = nullptr; + RuntimeProfile::Counter* plain_predicate_direct_rows = nullptr; RuntimeProfile::Counter* dict_filter_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_expr_rewrite_time = nullptr; RuntimeProfile::Counter* dict_filter_read_dict_time = nullptr; diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index c9d4c0380c674d..3b94371812dea0 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -18,8 +18,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -29,7 +31,10 @@ #include "core/block/block.h" #include "core/column/column_vector.h" #include "exprs/vcompound_pred.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" @@ -96,6 +101,108 @@ bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { namespace { +std::optional plain_predicate_op(std::string_view function_name, + bool reverse) { + PlainFixedPredicateOp op; + if (function_name == "eq") { + op = PlainFixedPredicateOp::EQ; + } else if (function_name == "ne") { + op = PlainFixedPredicateOp::NE; + } else if (function_name == "lt") { + op = PlainFixedPredicateOp::LT; + } else if (function_name == "le") { + op = PlainFixedPredicateOp::LE; + } else if (function_name == "gt") { + op = PlainFixedPredicateOp::GT; + } else if (function_name == "ge") { + op = PlainFixedPredicateOp::GE; + } else { + return std::nullopt; + } + if (!reverse) { + return op; + } + switch (op) { + case PlainFixedPredicateOp::LT: + return PlainFixedPredicateOp::GT; + case PlainFixedPredicateOp::LE: + return PlainFixedPredicateOp::GE; + case PlainFixedPredicateOp::GT: + return PlainFixedPredicateOp::LT; + case PlainFixedPredicateOp::GE: + return PlainFixedPredicateOp::LE; + case PlainFixedPredicateOp::EQ: + case PlainFixedPredicateOp::NE: + return op; + } + __builtin_unreachable(); +} + +std::optional> compile_plain_fixed_predicates( + const VExprContextSPtrs& conjuncts, const DataTypePtr& column_type, size_t block_position) { + const auto primitive_type = remove_nullable(column_type)->get_primitive_type(); + if (primitive_type != TYPE_INT && primitive_type != TYPE_BIGINT && + primitive_type != TYPE_FLOAT && primitive_type != TYPE_DOUBLE) { + return std::nullopt; + } + std::vector predicates; + predicates.reserve(conjuncts.size()); + for (const auto& conjunct : conjuncts) { + const auto* function = + conjunct == nullptr ? nullptr + : dynamic_cast(conjunct->root().get()); + if (function == nullptr || function->children().size() != 2) { + return std::nullopt; + } + const auto* left_slot = dynamic_cast(function->children()[0].get()); + const auto* right_slot = dynamic_cast(function->children()[1].get()); + const auto* left_literal = dynamic_cast(function->children()[0].get()); + const auto* right_literal = dynamic_cast(function->children()[1].get()); + const bool reverse = right_slot != nullptr && left_literal != nullptr; + const auto* slot = reverse ? right_slot : left_slot; + const auto* literal = reverse ? left_literal : right_literal; + if (slot == nullptr || literal == nullptr || + static_cast(slot->column_id()) != block_position) { + return std::nullopt; + } + if (!remove_nullable(literal->get_data_type())->equals(*remove_nullable(column_type))) { + return std::nullopt; + } + const auto op = plain_predicate_op(function->function_name(), reverse); + if (!op.has_value()) { + return std::nullopt; + } + Field value; + literal->get_column_ptr()->get(0, value); + if (value.is_null()) { + return std::nullopt; + } + switch (primitive_type) { + case TYPE_INT: + predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::INT32, *op, + value.get())); + break; + case TYPE_BIGINT: + predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::INT64, *op, + value.get())); + break; + case TYPE_FLOAT: + predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::FLOAT, *op, + value.get())); + break; + case TYPE_DOUBLE: + predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::DOUBLE, *op, + value.get())); + break; + default: + __builtin_unreachable(); + } + } + return predicates.empty() + ? std::nullopt + : std::optional>(std::move(predicates)); +} + detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request); @@ -1508,10 +1615,14 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, return Status::OK(); }; - auto read_predicate_column = [&](ParquetColumnReader* column_reader, size_t block_position, - ColumnId local_id, bool* used_dictionary_filter) -> Status { + auto read_predicate_column = + [&](ParquetColumnReader* column_reader, size_t block_position, ColumnId local_id, + const VExprContextSPtrs* single_column_conjuncts, bool* used_dictionary_filter, + bool* used_plain_filter) -> Status { DORIS_CHECK(used_dictionary_filter != nullptr); + DORIS_CHECK(used_plain_filter != nullptr); *used_dictionary_filter = false; + *used_plain_filter = false; DCHECK(remove_nullable(column_reader->type()) ->equals(*remove_nullable(file_block->get_by_position(block_position).type))) << column_reader->type()->get_name() << " " @@ -1552,6 +1663,47 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } } + if (single_column_conjuncts != nullptr && + request.is_predicate_only(format::LocalColumnId(cast_set(local_id)))) { + auto predicates = compile_plain_fixed_predicates( + *single_column_conjuncts, file_block->get_by_position(block_position).type, + block_position); + if (predicates.has_value()) { + const uint16_t selected_rows_before = *selected_rows; + IColumn::Filter compact_filter; + bool used_filter = false; + RETURN_IF_ERROR(column_reader->select_with_plain_filter( + *selection, *selected_rows, batch_rows, *predicates, &compact_filter, + &used_filter)); + if (used_filter) { + DORIS_CHECK_EQ(compact_filter.size(), selected_rows_before); + update_counter_if_not_null(_scan_profile.plain_predicate_direct_batches, 1); + update_counter_if_not_null(_scan_profile.plain_predicate_direct_rows, + selected_rows_before); + const uint16_t new_selected_rows = count_selected_rows(compact_filter); + const auto filtered_rows = static_cast(selected_rows_before) - + static_cast(new_selected_rows); + if (conjunct_filtered_rows != nullptr) { + *conjunct_filtered_rows += filtered_rows; + } + if (new_selected_rows != selected_rows_before) { + *selected_rows = apply_compact_filter_to_selection( + compact_filter, selection, selected_rows_before); + } + // Predicate-only values are dead after this exact comparison. Preserve only a + // row-shaped placeholder so later block positions keep their stable identity. + auto placeholder = column->clone_empty(); + placeholder->insert_many_defaults(*selected_rows); + file_block->replace_by_position(block_position, std::move(placeholder)); + read_column_positions.push_back(cast_set(block_position)); + remember_column_selection(cast_set(block_position)); + *predicate_columns_filtered = true; + *used_plain_filter = true; + return Status::OK(); + } + } + } + if (*selected_rows == batch_rows) { int64_t column_rows = 0; RETURN_IF_ERROR(column_reader->read(batch_rows, column, &column_rows)); @@ -1674,8 +1826,10 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, auto position_it = request.local_positions.find(format::LocalColumnId(fid)); DORIS_CHECK(position_it != request.local_positions.end()); bool used_dictionary_filter = false; + bool used_plain_filter = false; RETURN_IF_ERROR(read_predicate_column(column_reader.get(), position_it->second.value(), - fid, &used_dictionary_filter)); + fid, nullptr, &used_dictionary_filter, + &used_plain_filter)); } return Status::OK(); }; @@ -1714,16 +1868,21 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, predicate_batch_sequence); const int64_t start_ns = sample ? MonotonicNanos() : 0; bool used_dictionary_filter = false; - RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, - &used_dictionary_filter)); + bool used_plain_filter = false; const auto conjunct_it = schedule.single_column_conjuncts.find(block_position); + const VExprContextSPtrs* column_conjuncts = + conjunct_it == schedule.single_column_conjuncts.end() ? nullptr + : &conjunct_it->second; + RETURN_IF_ERROR(read_predicate_column(reader_it->second.get(), block_position, fid, + column_conjuncts, &used_dictionary_filter, + &used_plain_filter)); if (*selected_rows != 0 && conjunct_it != schedule.single_column_conjuncts.end()) { if (used_dictionary_filter) { const auto residual_it = _current_dictionary_residual_conjuncts.find(fid); DORIS_CHECK(residual_it != _current_dictionary_residual_conjuncts.end()); RETURN_IF_ERROR(execute_scheduled_dictionary_residual_conjuncts_with_profile( residual_it->second)); - } else { + } else if (!used_plain_filter) { RETURN_IF_ERROR(execute_scheduled_conjuncts_with_profile(conjunct_it->second)); } } diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 56c705fa12e518..2883dcd4f36c88 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -81,4 +81,15 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& name()); } +Status ParquetColumnReader::select_with_plain_filter(const SelectionVector&, uint16_t, int64_t, + const std::vector&, + IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *used_filter = false; + return Status::OK(); +} + } // namespace doris::format::parquet diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index b060c7a4ca71bd..59c0f0f63f33c7 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -22,6 +22,7 @@ #include "core/column/column.h" #include "core/data_type/data_type.h" #include "format_v2/parquet/parquet_profile.h" +#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "format_v2/parquet/selection_vector.h" namespace doris::format::parquet { @@ -60,6 +61,13 @@ class ParquetColumnReader { MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter); + // Consume batch_rows and evaluate eligible fixed-width PLAIN values without constructing a + // predicate column. Implementations must leave the cursor untouched when used_filter=false. + virtual Status select_with_plain_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const std::vector& predicates, + IColumn::Filter* row_filter, bool* used_filter); + // Native statistics are cumulative and can be recursively aggregated for complex columns. // Flush once at the scheduler batch boundary instead of snapshotting after each operation. virtual void flush_profile() {} diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index d8ae39b27f1d6e..2f508ca0aa3b85 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -47,6 +47,7 @@ #include "storage/cache/page_cache.h" #include "util/bit_util.h" #include "util/block_compression.h" +#include "util/cpu_info.h" #include "util/unaligned.h" namespace cctz { @@ -370,6 +371,20 @@ class SelectedDecodeSource final : public ParquetDecodeSource { return _decoder.decode_selected_dictionary_indices(_selection, indices); } + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _selection.selected_values); + return _decoder.decode_selected_dictionary_values(_selection, consumer); + } + + bool prefer_dictionary_index_materialization(size_t dictionary_bytes) const override { + // Avoid touching a dictionary larger than L2 for rows already removed by sparse selection. + // Cache-resident or dense batches instead fuse RLE decode and gather, eliminating the + // page-sized intermediate ID vector. + return _selection.selected_values < _selection.total_values && + dictionary_bytes > static_cast(CpuInfo::get_l2_cache_size()); + } + private: Decoder& _decoder; const ParquetSelection& _selection; @@ -445,16 +460,6 @@ bool visit_nullable_expandable_column(IColumn& column, Visitor&& visitor) { return false; } -bool has_expensive_nullable_sparse_materialization(const IColumn& column) { - return check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr || - check_and_get_column(&column) != nullptr; -} - template void expand_nullable_pod_values(ColumnType& column, size_t old_size, size_t compact_values, const NullMap& selected_nulls) { @@ -601,13 +606,36 @@ Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& ser selection.selected_values, selected_nulls); const bool expanded = visit_nullable_expandable_column(column, [&](auto& typed_column) { // Decode into the final nested column compactly, then expand in place. This preserves the - // V2 no-intermediate-column invariant while matching StarRocks' nullable sparse layout. + // V2 no-intermediate-column invariant while restoring the nullable sparse row layout. expand_nullable_values(typed_column, old_size, selection.selected_values, selected_nulls); }); DORIS_CHECK(expanded); return Status::OK(); } +class PlainPredicateConsumer final : public ParquetFixedValueConsumer { +public: + PlainPredicateConsumer(const std::vector& predicates, + IColumn::Filter* matches) + : _predicates(predicates), _matches(matches) { + DORIS_CHECK(_matches != nullptr); + } + + Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { + const size_t old_size = _matches->size(); + _matches->resize_fill(old_size + num_values, 1); + for (const auto& predicate : _predicates) { + RETURN_IF_ERROR(predicate.evaluate(values, num_values, value_width, + _matches->data() + old_size)); + } + return Status::OK(); + } + +private: + const std::vector& _predicates; + IColumn::Filter* _matches; +}; + } // namespace template @@ -1272,9 +1300,7 @@ Status ColumnChunkReader::materialize_values( state, select_vector, &_chunk_statistics.materialization_time); _chunk_statistics.hybrid_selection_ranges += state.selection.ranges.size(); - } else if ((context.encoding == ParquetValueEncoding::DICTIONARY || - has_expensive_nullable_sparse_materialization(*doris_column)) && - visit_nullable_expandable_column(*doris_column, [](auto&) {})) { + } else if (visit_nullable_expandable_column(*doris_column, [](auto&) {})) { // Parquet omits NULL leaf values from the physical stream. For example, the logical // DATE sequence [d0, NULL, d1, NULL, d2] is physically encoded as [d0, d1, d2]. The // generic fallback follows the logical selection runs, so those NULLs split one @@ -1286,9 +1312,9 @@ Status ColumnChunkReader::materialize_values( // Build selected non-NULL ranges in physical coordinates instead. In the example this // decodes [d0, d1, d2] compactly with one SerDe consumer, records [0, 1, 0, 1, 0] as // the logical NULL layout, and expands the final nested column backwards so unread - // compact values cannot be overwritten. Keep cheap non-decimal PLAIN numeric columns - // on the fallback: their skips are pointer arithmetic, while compact-and-expand would - // add a full output-column memory pass without amortizing expensive conversion work. + // compact values cannot be overwritten. Apply this to every expandable V2 scalar, + // including ordinary PLAIN primitives: otherwise nullable numeric predicates still + // fragment a physical span into millions of SerDe calls and defeat sparse decoding. ++_chunk_statistics.hybrid_selection_batches; status = decode_selected_nullable_values( *doris_column, serde, *_page_decoder, context, state, select_vector, @@ -1310,6 +1336,111 @@ Status ColumnChunkReader::materialize_values( return Status::OK(); } +template +bool ColumnChunkReader::can_filter_plain_values( + const std::vector& predicates) const { + if (predicates.empty() || _current_encoding != tparquet::Encoding::PLAIN || + (_metadata.type != tparquet::Type::INT32 && _metadata.type != tparquet::Type::INT64 && + _metadata.type != tparquet::Type::FLOAT && _metadata.type != tparquet::Type::DOUBLE)) { + return false; + } + const size_t physical_width = + _metadata.type == tparquet::Type::INT32 || _metadata.type == tparquet::Type::FLOAT + ? sizeof(uint32_t) + : sizeof(uint64_t); + return std::ranges::all_of(predicates, [&](const auto& predicate) { + const bool matching_type = (_metadata.type == tparquet::Type::INT32 && + predicate.type() == PlainFixedPredicateType::INT32) || + (_metadata.type == tparquet::Type::INT64 && + predicate.type() == PlainFixedPredicateType::INT64) || + (_metadata.type == tparquet::Type::FLOAT && + predicate.type() == PlainFixedPredicateType::FLOAT) || + (_metadata.type == tparquet::Type::DOUBLE && + predicate.type() == PlainFixedPredicateType::DOUBLE); + return matching_type && predicate.value_width() == physical_width; + }); +} + +template +Status ColumnChunkReader::filter_plain_values( + const std::vector& predicates, ColumnSelectVector& select_vector, + NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(selected_nulls != nullptr); + DORIS_CHECK(physical_matches != nullptr); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + *used_filter = false; + row_filter->clear(); + if (!can_filter_plain_values(predicates)) { + return Status::OK(); + } + if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { + return Status::IOError("Decode too many values in current page"); + } + + ParquetSelection selection; + selected_nulls->clear(); + selected_nulls->reserve(select_vector.num_values() - select_vector.num_filtered()); + size_t physical_cursor = 0; + auto build_selection = [&]() { + ColumnSelectVector::DataReadType read_type; + while (const size_t run_length = select_vector.get_next_run(&read_type)) { + switch (read_type) { + case ColumnSelectVector::CONTENT: + if (!selection.ranges.empty() && + selection.ranges.back().first + selection.ranges.back().count == + physical_cursor) { + selection.ranges.back().count += run_length; + } else { + selection.ranges.push_back({.first = physical_cursor, .count = run_length}); + } + selection.selected_values += run_length; + selected_nulls->resize_fill(selected_nulls->size() + run_length, 0); + physical_cursor += run_length; + break; + case ColumnSelectVector::NULL_DATA: + selected_nulls->resize_fill(selected_nulls->size() + run_length, 1); + break; + case ColumnSelectVector::FILTERED_CONTENT: + physical_cursor += run_length; + break; + case ColumnSelectVector::FILTERED_NULL: + break; + } + } + }; + if (select_vector.has_filter()) { + build_selection.template operator()(); + } else { + build_selection.template operator()(); + } + selection.total_values = physical_cursor; + DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); + DORIS_CHECK_EQ(selected_nulls->size(), + select_vector.num_values() - select_vector.num_filtered()); + + physical_matches->clear(); + if (selection.selected_values == 0) { + RETURN_IF_ERROR(_page_decoder->skip_values(selection.total_values)); + } else { + PlainPredicateConsumer consumer(predicates, physical_matches); + RETURN_IF_ERROR(_page_decoder->decode_selected_fixed_values(selection, consumer)); + DORIS_CHECK_EQ(physical_matches->size(), selection.selected_values); + } + + row_filter->reserve(selected_nulls->size()); + size_t physical_row = 0; + for (const uint8_t is_null : *selected_nulls) { + row_filter->push_back(is_null != 0 ? 0 : (*physical_matches)[physical_row++]); + } + DORIS_CHECK_EQ(physical_row, physical_matches->size()); + // Commit logical progress only after both the raw comparison and NULL remapping succeed. + _remaining_num_values -= select_vector.num_values(); + *used_filter = true; + return Status::OK(); +} + template Status ColumnChunkReader::seek_to_nested_row(size_t left_row) { if constexpr (OFFSET_INDEX) { diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 6bbcebb1bb9bd0..6a4b61b3e8b154 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -35,6 +35,7 @@ #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" +#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "util/slice.h" namespace doris { @@ -162,6 +163,14 @@ class ColumnChunkReader { ParquetDecodeContext& context, ParquetMaterializationState& state, ColumnSelectVector& select_vector); + // Evaluate selected fixed-width PLAIN values and return one keep byte per selected logical + // row. NULL comparisons are false and therefore never enter the physical consumer. + Status filter_plain_values(const std::vector& predicates, + ColumnSelectVector& select_vector, NullMap* selected_nulls, + IColumn::Filter* physical_matches, IColumn::Filter* row_filter, + bool* used_filter); + bool can_filter_plain_values(const std::vector& predicates) const; + // Get the repetition level decoder of current page. LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } // Get the definition level decoder of current page. diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index c2b8c443372600..3cfa8fead56f5a 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -1098,6 +1098,150 @@ Status ScalarColumnReader::_read_nested_column( return Status::OK(); } +template +Status ScalarColumnReader::_read_plain_filter_values( + size_t num_values, const std::vector& predicates, + FilterMap& filter_map, IColumn::Filter* row_filter) { + DORIS_CHECK(row_filter != nullptr); + _null_run_lengths.clear(); + if (_chunk_reader->max_def_level() > 0) { + LevelDecoder& def_decoder = _chunk_reader->def_level_decoder(); + size_t has_read = 0; + bool prev_is_null = true; + while (has_read < num_values) { + level_t def_level = -1; + const size_t loop_read = def_decoder.get_next_run(&def_level, num_values - has_read); + if (loop_read == 0) { + return Status::Corruption( + "Parquet definition level stream ended while filtering PLAIN values"); + } + const bool is_null = def_level < _field_schema->definition_level; + if (!(prev_is_null ^ is_null)) { + _null_run_lengths.emplace_back(0); + } + size_t remaining = loop_read; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + prev_is_null = is_null; + has_read += loop_read; + } + } else { + size_t remaining = num_values; + while (remaining > USHRT_MAX) { + _null_run_lengths.emplace_back(USHRT_MAX); + _null_run_lengths.emplace_back(0); + remaining -= USHRT_MAX; + } + _null_run_lengths.emplace_back(cast_set(remaining)); + } + RETURN_IF_ERROR(_select_vector.init(_null_run_lengths, num_values, nullptr, &filter_map, + _filter_map_index)); + _filter_map_index += num_values; + bool used_filter = false; + RETURN_IF_ERROR(_chunk_reader->filter_plain_values( + predicates, _select_vector, &_plain_predicate_nulls, &_plain_predicate_matches, + row_filter, &used_filter)); + // Chunk encodings are prevalidated before any definition level is consumed, so a false result + // here would make a materializing fallback observe an advanced level cursor. + DORIS_CHECK(used_filter); + return Status::OK(); +} + +template +Status ScalarColumnReader::read_plain_filter( + const std::vector& predicates, FilterMap& filter_map, + size_t batch_size, IColumn::Filter* row_filter, size_t* read_rows, bool* eof, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + if (_in_nested || predicates.empty()) { + return Status::OK(); + } + const auto source_primitive = remove_nullable(_field_schema->data_type)->get_primitive_type(); + const bool matching_source_type = + std::ranges::all_of(predicates, [&](const PlainFixedPredicate& predicate) { + return (source_primitive == TYPE_INT && + predicate.type() == PlainFixedPredicateType::INT32) || + (source_primitive == TYPE_BIGINT && + predicate.type() == PlainFixedPredicateType::INT64) || + (source_primitive == TYPE_FLOAT && + predicate.type() == PlainFixedPredicateType::FLOAT) || + (source_primitive == TYPE_DOUBLE && + predicate.type() == PlainFixedPredicateType::DOUBLE); + }); + if (!matching_source_type) { + return Status::OK(); + } + // Levels use RLE/BIT_PACKED, while every value page must be PLAIN. Reject mixed-encoding + // chunks before touching either cursor so the caller can safely use the normal expression path. + const bool plain_only = std::ranges::all_of( + _chunk_meta.meta_data.encodings, [](const tparquet::Encoding::type encoding) { + return encoding == tparquet::Encoding::PLAIN || + encoding == tparquet::Encoding::RLE || + encoding == tparquet::Encoding::BIT_PACKED; + }); + if (!plain_only) { + return Status::OK(); + } + + int64_t right_row = 0; + if constexpr (OFFSET_INDEX == false) { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + right_row = _chunk_reader->page_end_row(); + } else { + right_row = _chunk_reader->page_end_row(); + } + RowRanges read_ranges; + _generate_read_ranges(RowRange {_current_row_index, right_row}, &read_ranges); + if (read_ranges.count() == 0) { + _current_row_index = right_row; + } else { + RETURN_IF_ERROR(_chunk_reader->parse_page_header()); + RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); + if (!_chunk_reader->can_filter_plain_values(predicates)) { + return Status::OK(); + } + size_t has_read = 0; + for (size_t idx = 0; idx < read_ranges.range_size(); ++idx) { + const auto range = read_ranges.get_range(idx); + const size_t skip_values = range.from() - _current_row_index; + RETURN_IF_ERROR(_skip_values(skip_values)); + _current_row_index += skip_values; + const size_t values = + std::min(static_cast(range.to() - range.from()), batch_size - has_read); + IColumn::Filter fragment_filter; + RETURN_IF_ERROR( + _read_plain_filter_values(values, predicates, filter_map, &fragment_filter)); + row_filter->insert(row_filter->end(), fragment_filter.begin(), fragment_filter.end()); + has_read += values; + *read_rows += values; + _current_row_index += values; + if (has_read == batch_size) { + break; + } + } + } + + if (right_row == _current_row_index) { + if (!_chunk_reader->has_next_page()) { + *eof = true; + } else { + RETURN_IF_ERROR(_chunk_reader->next_page()); + } + } + *used_filter = true; + return Status::OK(); +} + template Status ScalarColumnReader::read_column_levels(FilterMap& filter_map, size_t batch_size, diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 22683a70e5ac6f..98be7df08c8b65 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -34,6 +34,7 @@ #include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native/common.h" +#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader_writer_fwd.h" @@ -189,6 +190,21 @@ class ColumnReader { bool* eof, bool is_dict_filter, int64_t real_column_size = -1) = 0; + // Evaluate a predicate-only scalar directly from fixed-width PLAIN page bytes. The default is + // a non-consuming fallback for nested and synthetic readers. + virtual Status read_plain_filter(const std::vector&, FilterMap&, size_t, + IColumn::Filter* row_filter, size_t* read_rows, bool* eof, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(read_rows != nullptr); + DORIS_CHECK(eof != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *read_rows = 0; + *used_filter = false; + return Status::OK(); + } + // Consume a nested batch while retaining only definition/repetition levels. This is used when // schema evolution makes every projected STRUCT child synthetic: the parent still needs one // physical leaf's shape, but decoding that leaf's strings or other payload would be wasted. @@ -267,6 +283,9 @@ class ScalarColumnReader : public ColumnReader { const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override; + Status read_plain_filter(const std::vector& predicates, + FilterMap& filter_map, size_t batch_size, IColumn::Filter* row_filter, + size_t* read_rows, bool* eof, bool* used_filter) override; Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof) override; Result convert_dict_column_to_string_column( @@ -375,6 +394,8 @@ class ScalarColumnReader : public ColumnReader { std::vector _null_run_lengths; std::unordered_set _ancestor_null_indices; std::vector _nested_filter_map_data; + NullMap _plain_predicate_nulls; + IColumn::Filter _plain_predicate_matches; FilterMap _nested_filter_map; ColumnSelectVector _select_vector; int64_t _convert_time = 0; @@ -387,6 +408,9 @@ class ScalarColumnReader : public ColumnReader { Status _skip_values(size_t num_values); Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, bool is_dict_filter); + Status _read_plain_filter_values(size_t num_values, + const std::vector& predicates, + FilterMap& filter_map, IColumn::Filter* row_filter); Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter); diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index 830f86e204bf3c..e3fa2e4fec8358 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -215,6 +215,29 @@ class BaseDictDecoder : public Decoder { return Status::OK(); } + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + return _decode_dictionary_values(num_values, 0, dictionary_size(), consumer); + } + + Status decode_selected_dictionary_values( + const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) override { + const size_t num_dictionary_values = dictionary_size(); + size_t cursor = 0; + for (const auto& range : selection.ranges) { + DORIS_CHECK(range.first >= cursor); + RETURN_IF_ERROR(_decode_and_validate_skipped(range.first - cursor, cursor, + num_dictionary_values)); + RETURN_IF_ERROR(_decode_dictionary_values(range.count, range.first, + num_dictionary_values, consumer)); + cursor = range.first + range.count; + } + DORIS_CHECK(cursor <= selection.total_values); + return _decode_and_validate_skipped(selection.total_values - cursor, cursor, + num_dictionary_values); + } + void release_scratch(size_t max_retained_bytes) override { release_vector_if_oversized(&_skip_indices, max_retained_bytes); } @@ -263,6 +286,57 @@ class BaseDictDecoder : public Decoder { return Status::OK(); } + Status _decode_dictionary_values(size_t num_values, size_t row_offset, + size_t num_dictionary_values, + ParquetDictionaryValueConsumer& consumer) { + constexpr size_t kLiteralBatchSize = 1024; + size_t decoded_values = 0; + while (decoded_values < num_values) { + const int32_t repeats = _index_batch_decoder->NextNumRepeats(); + if (repeats > 0) { + const size_t run = std::min(repeats, num_values - decoded_values); + const uint32_t index = + _index_batch_decoder->GetRepeatedValue(cast_set(run)); + if (UNLIKELY(static_cast(index) >= num_dictionary_values)) { + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + index, row_offset + decoded_values, num_dictionary_values); + } + RETURN_IF_ERROR(consumer.consume_repeated(index, run)); + decoded_values += run; + continue; + } + + const int32_t literals = _index_batch_decoder->NextNumLiterals(); + if (UNLIKELY(literals == 0)) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + const size_t batch = std::min( + {static_cast(literals), num_values - decoded_values, + kLiteralBatchSize}); + _skip_indices.resize(batch); + if (UNLIKELY(!_index_batch_decoder->GetLiteralValues(cast_set(batch), + _skip_indices.data()))) { + return Status::IOError("Can't read enough Parquet dictionary indices"); + } + if (UNLIKELY(!dictionary_indices_in_bounds(_skip_indices.data(), batch, + num_dictionary_values))) { + for (size_t row = 0; row < batch; ++row) { + if (_skip_indices[row] < num_dictionary_values) { + continue; + } + return Status::Corruption( + "Parquet dictionary index {} at row {} exceeds dictionary size {}", + _skip_indices[row], row_offset + decoded_values + row, + num_dictionary_values); + } + } + RETURN_IF_ERROR(consumer.consume_indices(_skip_indices.data(), batch)); + decoded_values += batch; + } + return Status::OK(); + } + // For dictionary encoding DorisUniqueBufferPtr _dict; std::unique_ptr> _index_batch_decoder; diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index c191ee50e09b90..64506ccf604eaf 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -337,6 +337,60 @@ Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_ return Status::OK(); } +Status NativeColumnReader::read_with_plain_filter( + int64_t rows, const uint8_t* filter_data, bool filter_all, + const std::vector& predicates, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter) { + DORIS_CHECK(rows >= 0); + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(rows_read != nullptr); + DORIS_CHECK(used_filter != nullptr); + row_filter->clear(); + *rows_read = 0; + *used_filter = false; + if (rows == 0) { + return Status::OK(); + } + + native::FilterMap filter; + RETURN_IF_ERROR(filter.init(filter_data, static_cast(rows), filter_all)); + _native_reader->reset_filter_map_index(); + bool eof = false; + int64_t consecutive_empty_calls = 0; + while (*rows_read < rows && !eof) { + size_t loop_rows = 0; + IColumn::Filter loop_filter; + bool loop_used = false; + RETURN_IF_ERROR(_native_reader->read_plain_filter( + predicates, filter, static_cast(rows - *rows_read), &loop_filter, + &loop_rows, &eof, &loop_used)); + if (!loop_used) { + // A fallback is safe only before this call has consumed a logical row. Plain-only + // chunk validation makes the decision stable for all later page fragments. + DORIS_CHECK_EQ(*rows_read, 0); + row_filter->clear(); + return Status::OK(); + } + row_filter->insert(row_filter->end(), loop_filter.begin(), loop_filter.end()); + if (loop_rows == 0 && !eof) { + if (++consecutive_empty_calls > _row_group_rows + 1) { + return Status::Corruption( + "Native parquet PLAIN predicate made no progress for column {}", _name); + } + continue; + } + consecutive_empty_calls = 0; + *rows_read += static_cast(loop_rows); + } + if (*rows_read != rows) { + return Status::Corruption( + "Native parquet PLAIN predicate returned {} rows, expected {} for {}", *rows_read, + rows, _name); + } + *used_filter = true; + return Status::OK(); +} + Status NativeColumnReader::validate_selected_span(int64_t rows) { DORIS_CHECK(rows >= 0); while (_selected_range_idx < _selected_ranges.size()) { @@ -530,6 +584,34 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& return Status::OK(); } +Status NativeColumnReader::select_with_plain_filter( + const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, + const std::vector& predicates, IColumn::Filter* row_filter, + bool* used_filter) { + DORIS_CHECK(row_filter != nullptr); + DORIS_CHECK(used_filter != nullptr); + RETURN_IF_ERROR(validate_selected_span(batch_rows)); + RETURN_IF_ERROR(selection.verify(selected_rows, batch_rows)); + const uint8_t* filter_data = nullptr; + RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); + int64_t rows_read = 0; + RETURN_IF_ERROR(read_with_plain_filter(batch_rows, filter_data, selected_rows == 0, predicates, + row_filter, &rows_read, used_filter)); + if (!*used_filter) { + return Status::OK(); + } + DORIS_CHECK_EQ(rows_read, batch_rows); + if (row_filter->size() != selected_rows) { + return Status::Corruption( + "Native parquet PLAIN predicate returned {} selected rows, expected {} for {}", + row_filter->size(), selected_rows, _name); + } + advance_selected_span(rows_read); + update_reader_read_rows(selected_rows); + update_reader_skip_rows(batch_rows - selected_rows); + return Status::OK(); +} + void NativeColumnReader::flush_profile() { record_page_fragments(sync_native_profile()); } diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 5bfe0ccc15aa4a..9716f1adb180b8 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -88,6 +88,10 @@ class NativeColumnReader final : public ParquetColumnReader { const IColumn::Filter& dictionary_filter, MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter) override; + Status select_with_plain_filter(const SelectionVector& selection, uint16_t selected_rows, + int64_t batch_rows, + const std::vector& predicates, + IColumn::Filter* row_filter, bool* used_filter) override; void flush_profile() override; bool crossed_page_since_last_batch() override; Result dictionary_values() override; @@ -108,6 +112,10 @@ class NativeColumnReader final : public ParquetColumnReader { Status read_with_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, MutableColumnPtr& column, const DataTypePtr& output_type, bool dictionary_ids, int64_t* rows_read); + Status read_with_plain_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, + const std::vector& predicates, + IColumn::Filter* row_filter, int64_t* rows_read, + bool* used_filter); int64_t sync_native_profile(); void record_page_fragments(int64_t page_fragments); Status validate_selected_span(int64_t rows); diff --git a/be/src/format_v2/parquet/reader/plain_fixed_predicate.h b/be/src/format_v2/parquet/reader/plain_fixed_predicate.h new file mode 100644 index 00000000000000..0596ff6718d5c8 --- /dev/null +++ b/be/src/format_v2/parquet/reader/plain_fixed_predicate.h @@ -0,0 +1,125 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "util/unaligned.h" + +namespace doris::format::parquet { + +enum class PlainFixedPredicateOp : uint8_t { EQ, NE, LT, LE, GT, GE }; +enum class PlainFixedPredicateType : uint8_t { INT32, INT64, FLOAT, DOUBLE }; + +// A compiled predicate over the physical little-endian values of a fixed-width PLAIN page. +// Compilation is deliberately restricted to expressions whose Doris value and Parquet physical +// value have identical comparison semantics; casts and logical-type conversions stay on the +// ordinary materialization path. +class PlainFixedPredicate { +public: + template + static PlainFixedPredicate create(PlainFixedPredicateType type, PlainFixedPredicateOp op, + T literal) { + PlainFixedPredicate predicate; + predicate._type = type; + predicate._op = op; + static_assert(sizeof(T) <= sizeof(predicate._literal)); + memcpy(predicate._literal.data(), &literal, sizeof(T)); + return predicate; + } + + size_t value_width() const { + switch (_type) { + case PlainFixedPredicateType::INT32: + case PlainFixedPredicateType::FLOAT: + return sizeof(uint32_t); + case PlainFixedPredicateType::INT64: + case PlainFixedPredicateType::DOUBLE: + return sizeof(uint64_t); + } + __builtin_unreachable(); + } + + PlainFixedPredicateType type() const { return _type; } + + // AND this predicate into matches. The caller owns NULL handling because Parquet omits NULLs + // from the physical value stream. + Status evaluate(const uint8_t* values, size_t num_values, size_t value_width, + uint8_t* matches) const { + if (UNLIKELY(value_width != this->value_width())) { + return Status::Corruption("PLAIN predicate width {} does not match expected {}", + value_width, this->value_width()); + } + switch (_type) { + case PlainFixedPredicateType::INT32: + return _evaluate(values, num_values, matches); + case PlainFixedPredicateType::INT64: + return _evaluate(values, num_values, matches); + case PlainFixedPredicateType::FLOAT: + return _evaluate(values, num_values, matches); + case PlainFixedPredicateType::DOUBLE: + return _evaluate(values, num_values, matches); + } + __builtin_unreachable(); + } + +private: + template + Status _evaluate(const uint8_t* values, size_t num_values, uint8_t* matches) const { + const T literal = unaligned_load(_literal.data()); + for (size_t row = 0; row < num_values; ++row) { + if (matches[row] == 0) { + continue; + } + const T value = unaligned_load(values + row * sizeof(T)); + bool keep = false; + switch (_op) { + case PlainFixedPredicateOp::EQ: + keep = value == literal; + break; + case PlainFixedPredicateOp::NE: + keep = value != literal; + break; + case PlainFixedPredicateOp::LT: + keep = value < literal; + break; + case PlainFixedPredicateOp::LE: + keep = value <= literal; + break; + case PlainFixedPredicateOp::GT: + keep = value > literal; + break; + case PlainFixedPredicateOp::GE: + keep = value >= literal; + break; + } + matches[row] = static_cast(keep); + } + return Status::OK(); + } + + PlainFixedPredicateType _type = PlainFixedPredicateType::INT32; + PlainFixedPredicateOp _op = PlainFixedPredicateOp::EQ; + std::array _literal {}; +}; + +} // namespace doris::format::parquet diff --git a/be/src/util/cpu_info.cpp b/be/src/util/cpu_info.cpp index 400c8f9381b77c..8c24a886acb2b4 100644 --- a/be/src/util/cpu_info.cpp +++ b/be/src/util/cpu_info.cpp @@ -377,6 +377,17 @@ void CpuInfo::_get_cache_info(long cache_sizes[NUM_CACHE_LEVELS], #endif } +long CpuInfo::get_l2_cache_size() { + static const long l2_cache_size = [] { + long cache_sizes[NUM_CACHE_LEVELS] = {}; + long cache_line_sizes[NUM_CACHE_LEVELS] = {}; + _get_cache_info(cache_sizes, cache_line_sizes); + constexpr long DEFAULT_L2_CACHE_SIZE = 256 * 1024; + return cache_sizes[L2_CACHE] > 0 ? cache_sizes[L2_CACHE] : DEFAULT_L2_CACHE_SIZE; + }(); + return l2_cache_size; +} + std::string CpuInfo::debug_string() { DCHECK(initialized_); std::stringstream stream; diff --git a/be/src/util/cpu_info.h b/be/src/util/cpu_info.h index 19548ecc964659..c75b814721cc57 100644 --- a/be/src/util/cpu_info.h +++ b/be/src/util/cpu_info.h @@ -146,6 +146,10 @@ class CpuInfo { return model_name_; } + // Some virtualized hosts do not expose cache topology. Always return a positive value so hot + // decode paths can use this threshold without repeating platform-specific guards. + static long get_l2_cache_size(); + static std::string debug_string(); /// A utility class for temporarily disabling CPU features. Usage: diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 9b9f7b511118be..715312ebaa2831 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -42,6 +42,7 @@ #include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" +#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "io/fs/buffered_reader.h" #include "util/block_compression.h" #include "util/coding.h" @@ -53,6 +54,26 @@ namespace doris::format::parquet::native { namespace { +TEST(ParquetV2NativeDecoderTest, PlainFixedPredicateEvaluatesPhysicalValuesWithoutColumn) { + const std::array values = {1, 4, 7, 10, 13, 16}; + std::array matches {}; + matches.fill(1); + const auto greater_equal = PlainFixedPredicate::create(PlainFixedPredicateType::INT32, + PlainFixedPredicateOp::GE, int32_t {7}); + const auto less_than = PlainFixedPredicate::create(PlainFixedPredicateType::INT32, + PlainFixedPredicateOp::LT, int32_t {16}); + + ASSERT_TRUE(greater_equal + .evaluate(reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), matches.data()) + .ok()); + ASSERT_TRUE(less_than + .evaluate(reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1, 1, 0})); +} + class RejectFixedConsumer final : public ParquetFixedValueConsumer { public: Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { @@ -119,6 +140,45 @@ class CaptureFixedConsumer final : public ParquetFixedValueConsumer { std::vector bytes; }; +class ScriptedDictionaryMaterializationSource final : public ParquetDecodeSource { +public: + ScriptedDictionaryMaterializationSource(std::vector ids, bool prefer_indices) + : _ids(std::move(ids)), _prefer_indices(prefer_indices) {} + + Status decode_fixed_values(size_t, ParquetFixedValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no fixed values"); + } + + Status decode_binary_values(size_t, ParquetBinaryValueConsumer&) override { + return Status::NotSupported("scripted dictionary source has no binary values"); + } + + Status skip_values(size_t) override { return Status::OK(); } + + Status decode_dictionary_indices(size_t num_values, std::vector* indices) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++index_batches; + *indices = _ids; + return Status::OK(); + } + + Status decode_dictionary_values(size_t num_values, + ParquetDictionaryValueConsumer& consumer) override { + DORIS_CHECK_EQ(num_values, _ids.size()); + ++direct_batches; + return consumer.consume_indices(_ids.data(), _ids.size()); + } + + bool prefer_dictionary_index_materialization(size_t) const override { return _prefer_indices; } + + size_t direct_batches = 0; + size_t index_batches = 0; + +private: + std::vector _ids; + bool _prefer_indices; +}; + const RowRanges& scripted_row_ranges() { static const RowRanges ranges; return ranges; @@ -606,7 +666,65 @@ Status materialize_selected_dictionary_strings(const std::vector& d return Status::OK(); } -TEST(ParquetV2NativeDecoderTest, NullableSparsePlainSelectionRetainsCheaperRangeFallback) { +TEST(ParquetV2NativeDecoderTest, PlainFixedPredicateMapsNullableSparseRowsDirectly) { + const std::vector physical_values {1, 4, 7, 10, 13}; + constexpr size_t LOGICAL_VALUES = 7; + tparquet::PageHeader header; + header.type = tparquet::PageType::DATA_PAGE; + header.__set_compressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__set_uncompressed_page_size(physical_values.size() * sizeof(int32_t)); + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(LOGICAL_VALUES); + header.data_page_header.__set_encoding(tparquet::Encoding::PLAIN); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + std::vector payload(physical_values.size() * sizeof(int32_t)); + memcpy(payload.data(), physical_values.data(), payload.size()); + auto bytes = serialize_page(header, payload); + + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(tparquet::Type::INT32); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(LOGICAL_VALUES); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = tparquet::Type::INT32; + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, LOGICAL_VALUES, + nullptr, page_context); + ASSERT_TRUE(chunk_reader.init().ok()); + ASSERT_TRUE(chunk_reader.load_page_data().ok()); + + const std::vector null_runs {1, 1, 2, 1, 2}; + const std::vector input_filter {1, 0, 1, 1, 1, 0, 1}; + FilterMap filter; + ASSERT_TRUE(filter.init(input_filter.data(), input_filter.size(), false).ok()); + ColumnSelectVector select_vector; + ASSERT_TRUE(select_vector.init(null_runs, LOGICAL_VALUES, nullptr, &filter, 0).ok()); + const std::vector predicates { + PlainFixedPredicate::create(PlainFixedPredicateType::INT32, PlainFixedPredicateOp::GE, + int32_t {4}), + PlainFixedPredicate::create(PlainFixedPredicateType::INT32, PlainFixedPredicateOp::LT, + int32_t {13})}; + NullMap selected_nulls; + IColumn::Filter physical_matches; + IColumn::Filter row_filter; + bool used_filter = false; + ASSERT_TRUE(chunk_reader + .filter_plain_values(predicates, select_vector, &selected_nulls, + &physical_matches, &row_filter, &used_filter) + .ok()); + + EXPECT_TRUE(used_filter); + EXPECT_EQ(selected_nulls, (NullMap {0, 0, 0, 1, 0})); + EXPECT_EQ(physical_matches, (IColumn::Filter {0, 1, 1, 0})); + EXPECT_EQ(row_filter, (IColumn::Filter {0, 1, 1, 0, 0})); + EXPECT_EQ(chunk_reader.remaining_num_values(), 0); +} + +TEST(ParquetV2NativeDecoderTest, NullableSparsePlainPrimitiveSelectionBatchesPhysicalPayload) { constexpr size_t LOGICAL_VALUES = 4095; std::vector null_runs(LOGICAL_VALUES, 1); std::vector physical_values((LOGICAL_VALUES + 1) / 2); @@ -629,10 +747,9 @@ TEST(ParquetV2NativeDecoderTest, NullableSparsePlainSelectionRetainsCheaperRange EXPECT_EQ(assert_cast(*column).get_data(), (ColumnInt32::Container {-7, 0, 500, 0, 1000, 0})); EXPECT_EQ(null_map, (NullMap {0, 0, 0, 1, 0, 1})); - // Fixed-width PLAIN skips are pointer arithmetic, so compact-and-expand would add work. - EXPECT_EQ(statistics.hybrid_selection_batches, 0); - EXPECT_EQ(statistics.hybrid_selection_ranges, 0); - EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_ranges, 3); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); } TEST(ParquetV2NativeDecoderTest, NullableSparsePlainDecimalSelectionBatchesPhysicalPayload) { @@ -729,7 +846,7 @@ TEST(ParquetV2NativeDecoderTest, NullableSparseDictionarySelectionBatchesPhysica constexpr size_t LOGICAL_VALUES = 4095; std::vector dictionary(16); std::iota(dictionary.begin(), dictionary.end(), 100); - // One value followed by nineteen NULLs exercises StarRocks' sparse-null threshold (<10%). + // One value followed by nineteen NULLs exercises the sparse-null threshold (<10%). std::vector null_runs; for (size_t row = 0; row < LOGICAL_VALUES;) { null_runs.push_back(1); @@ -780,6 +897,105 @@ TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationSkipsFailureScanWhenDi EXPECT_EQ(state.dictionary_failure_scan_rows, 0); } +TEST(ParquetV2NativeDecoderTest, DictionaryMaterializationUsesCacheAwareExecutionShape) { + auto verify_strategy = [](bool prefer_indices, + ParquetDictionaryMaterializationStrategy expected_strategy, + size_t expected_direct_batches, size_t expected_index_batches) { + ParquetMaterializationState state; + state.typed_dictionary = ColumnInt32::create(); + assert_cast(*state.typed_dictionary).get_data() = {10, 20, 30, 40}; + ScriptedDictionaryMaterializationSource source({3, 0, 2, 1, 3}, prefer_indices); + auto output = ColumnInt32::create(); + + const auto status = state.materialize_dictionary(*output, source, 5); + EXPECT_TRUE(status.ok()) << status; + EXPECT_EQ(output->get_data(), (ColumnInt32::Container {40, 10, 30, 20, 40})); + EXPECT_EQ(state.dictionary_materialization_strategy, expected_strategy); + EXPECT_EQ(source.direct_batches, expected_direct_batches); + EXPECT_EQ(source.index_batches, expected_index_batches); + }; + + verify_strategy(false, ParquetDictionaryMaterializationStrategy::DIRECT, 1, 0); + verify_strategy(true, ParquetDictionaryMaterializationStrategy::INDICES, 0, 1); +} + +TEST(ParquetV2NativeDecoderTest, DictionaryRepeatedRunsGatherDirectlyIntoDestination) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 1); + for (size_t row = 0; row < 64; ++row) { + encoder.Put(1); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 1; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + ASSERT_TRUE( + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 64, state).ok()); + EXPECT_EQ(state.dictionary_materialization_strategy, + ParquetDictionaryMaterializationStrategy::DIRECT); + ASSERT_EQ(output->size(), 64); + for (size_t row = 0; row < output->size(); ++row) { + EXPECT_EQ(assert_cast(*output).get_element(row), 20); + } +} + +TEST(ParquetV2NativeDecoderTest, DictionaryDirectGatherRollsBackLateCorruptRun) { + const std::array dictionary_values {10, 20}; + auto dictionary = make_unique_buffer(sizeof(dictionary_values)); + memcpy(dictionary.get(), dictionary_values.data(), sizeof(dictionary_values)); + std::unique_ptr decoder; + ASSERT_TRUE( + Decoder::get_decoder(tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY, decoder) + .ok()); + decoder->set_type_length(sizeof(int32_t)); + ASSERT_TRUE(decoder->set_dict(dictionary, sizeof(dictionary_values), dictionary_values.size()) + .ok()); + + faststring encoded_ids; + RleEncoder encoder(&encoded_ids, 2); + for (size_t row = 0; row < 8; ++row) { + encoder.Put(0); + } + for (size_t row = 0; row < 8; ++row) { + encoder.Put(3); + } + encoder.Flush(); + std::vector payload(encoded_ids.size() + 1); + payload[0] = 2; + memcpy(payload.data() + 1, encoded_ids.data(), encoded_ids.size()); + Slice id_slice(payload.data(), payload.size()); + ASSERT_TRUE(decoder->set_data(&id_slice).ok()); + + DataTypeInt32 type; + auto output = type.create_column(); + ParquetMaterializationState state; + ParquetDecodeContext context {.physical_type = ParquetPhysicalType::INT32, + .encoding = ParquetValueEncoding::DICTIONARY}; + const auto status = + type.get_serde()->read_column_from_parquet(*output, *decoder, context, 16, state); + EXPECT_TRUE(status.is()) << status; + EXPECT_TRUE(output->empty()); +} + TEST(ParquetV2NativeDecoderTest, DictionaryIndexBoundsCheckHandlesVectorTailAndUnsignedIds) { const std::vector valid {0, 7, 1, 6, 2, 5, 3, 4, 7, 0, 6}; EXPECT_TRUE(native::dictionary_indices_in_bounds(valid.data(), valid.size(), 8)); @@ -865,9 +1081,9 @@ TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes EXPECT_EQ(assert_cast(*all_null_column).get_data(), (ColumnInt32::Container {8, 0, 0, 0})); EXPECT_EQ(all_null_map, (NullMap {0, 1, 1, 1})); - EXPECT_EQ(statistics.hybrid_selection_batches, 0); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); EXPECT_EQ(statistics.hybrid_selection_ranges, 0); - EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); auto dictionary_all_null_column = type.create_column(); assert_cast(*dictionary_all_null_column).get_data().push_back(8); @@ -897,9 +1113,9 @@ TEST(ParquetV2NativeDecoderTest, NullableSparseSelectionHandlesEmptyOutputShapes EXPECT_EQ(assert_cast(*all_filtered_column).get_data(), (ColumnInt32::Container {9})); EXPECT_EQ(all_filtered_map, (NullMap {0})); - EXPECT_EQ(statistics.hybrid_selection_batches, 0); + EXPECT_EQ(statistics.hybrid_selection_batches, 1); EXPECT_EQ(statistics.hybrid_selection_ranges, 0); - EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 1); + EXPECT_EQ(statistics.hybrid_selection_null_fallback_batches, 0); auto dictionary_all_filtered_column = type.create_column(); assert_cast(*dictionary_all_filtered_column).get_data().push_back(9); diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index f047337e97e7e9..096dfaa24435b4 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -44,8 +44,10 @@ #include "core/data_type/data_type_number.h" #include "core/data_type/data_type_string.h" #include "core/field.h" +#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr.h" #include "exprs/vexpr_context.h" +#include "exprs/vliteral.h" #include "exprs/vslot_ref.h" #include "format_v2/expr/delete_predicate.h" #include "format_v2/file_reader.h" @@ -259,6 +261,38 @@ VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr:: return VExprContext::create_shared(std::make_shared(column_id, op, value)); } +VExprContextSPtr create_int32_function_conjunct(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t value) { + const auto int_type = std::make_shared(); + const auto nullable_int_type = make_nullable(int_type); + const auto result_type = make_nullable(std::make_shared()); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({nullable_int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(result_type->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(result_type->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(true); + auto root = VectorizedFnCall::create_shared(node); + root->add_child( + VSlotRef::create_shared(column_id, column_id, -1, nullable_int_type, "plain_id")); + root->add_child(VLiteral::create_shared(int_type, Field::create_field(value))); + auto context = VExprContext::create_shared(std::move(root)); + // Direct evaluation does not execute the expression, but a fallback must still fail this test + // instead of silently using an unprepared test-only context. + context->_prepared = true; + context->_opened = true; + return context; +} + VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_column_id, int32_t upper_bound) { return VExprContext::create_shared( @@ -1131,6 +1165,37 @@ TEST_F(ParquetScanTest, PredicateOnlyColumnDropsPayloadAfterFiltering) { EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); } +TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 2)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 4); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30, 40, 50, 60})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectBatches"), 1); + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectRows"), 6); + EXPECT_EQ(counter_value(profile, "PredicateCompactionCount"), 0); + EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); +} + // Scenario: every physical batch in every row group is rejected. Predicate readers reach each row // group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that // reader and its pending lag instead of issuing SkipRecords for values that can never be observed. diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index 4bebb6da200e16..a832b964ad8690 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -276,6 +276,104 @@ TEST(ParquetSchemaTest, NativeSchemaBoundsRecursiveDepth) { rejected.parse_from_thrift(nested_native_schema(MAX_NATIVE_SCHEMA_DEPTH + 1)).ok()); } +TEST(ParquetSchemaTest, NativeListTupleCompatibilityRequiresEnclosingListName) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto list = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + schema.__set_converted_type(tparquet::ConvertedType::LIST); + return schema; + }; + auto wrapper = [](const std::string& name) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + return schema; + }; + auto item = []() { + tparquet::SchemaElement schema; + schema.__set_name("item"); + schema.__set_type(tparquet::Type::INT32); + schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + return schema; + }; + + NativeFieldDescriptor mismatched; + ASSERT_TRUE(mismatched.parse_from_thrift({root(), list("xs"), wrapper("other_tuple"), item()}) + .ok()); + const auto* mismatched_element = &mismatched.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(mismatched_element->data_type)->get_primitive_type(), TYPE_INT); + + NativeFieldDescriptor matching; + ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}) + .ok()); + const auto* matching_element = &matching.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(matching_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(matching_element->children.size(), 1); + EXPECT_EQ(matching_element->children[0].name, "item"); +} + +TEST(ParquetSchemaTest, NativeListPreservesRepeatedAndAnnotatedElementWrappers) { + auto root = []() { + tparquet::SchemaElement schema; + schema.__set_name("schema"); + schema.__set_num_children(1); + return schema; + }; + auto group = [](const std::string& name, tparquet::FieldRepetitionType::type repetition) { + tparquet::SchemaElement schema; + schema.__set_name(name); + schema.__set_num_children(1); + schema.__set_repetition_type(repetition); + return schema; + }; + auto outer_list = group("outer", tparquet::FieldRepetitionType::OPTIONAL); + outer_list.__set_converted_type(tparquet::ConvertedType::LIST); + + auto repeated_wrapper = group("list", tparquet::FieldRepetitionType::REPEATED); + tparquet::SchemaElement repeated_items; + repeated_items.__set_name("items"); + repeated_items.__set_type(tparquet::Type::INT32); + repeated_items.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); + NativeFieldDescriptor repeated; + ASSERT_TRUE(repeated + .parse_from_thrift( + {root(), outer_list, repeated_wrapper, repeated_items}) + .ok()); + const auto* repeated_element = &repeated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(repeated_element->data_type)->get_primitive_type(), TYPE_STRUCT); + ASSERT_EQ(repeated_element->children.size(), 1); + EXPECT_EQ(repeated_element->children[0].name, "items"); + EXPECT_EQ(remove_nullable(repeated_element->children[0].data_type)->get_primitive_type(), + TYPE_ARRAY); + + auto annotated_wrapper = repeated_wrapper; + annotated_wrapper.__set_converted_type(tparquet::ConvertedType::LIST); + auto nested_wrapper = repeated_wrapper; + tparquet::SchemaElement value; + value.__set_name("value"); + value.__set_type(tparquet::Type::INT32); + value.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + NativeFieldDescriptor annotated; + ASSERT_TRUE(annotated + .parse_from_thrift( + {root(), outer_list, annotated_wrapper, nested_wrapper, value}) + .ok()); + const auto* annotated_element = &annotated.get_column(0)->children[0]; + EXPECT_EQ(remove_nullable(annotated_element->data_type)->get_primitive_type(), TYPE_ARRAY); + ASSERT_EQ(annotated_element->children.size(), 1); + EXPECT_EQ(remove_nullable(annotated_element->children[0].data_type)->get_primitive_type(), + TYPE_INT); +} + TEST(ParquetSchemaTest, NativeMetadataRejectsRowGroupChunkCardinalityAndMissingMetadata) { auto make_metadata = []() { tparquet::FileMetaData metadata; diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index 67c7f226cd6401..134240c75e266f 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -150,14 +150,21 @@ format-specific checklist when reviewing Parquet or ORC. - Check direct materialization for PLAIN, RLE/dictionary, DELTA_BINARY_PACKED, DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded values without allocating output; null runs must append defaults without advancing payload. -- For a filtered page fragment without definition-level NULLs, require one SerDe entry and one - batch-level selected-decode dispatch. Selection ranges belong to persistent reader scratch; - per-range virtual SerDe/decoder calls in the hot path are a review blocker. Fixed PLAIN should +- For predicate-only fixed-width PLAIN primitives, allow direct comparison only after proving the + whole Column Chunk uses compatible PLAIN value pages and the expression has identical physical + comparison semantics. The fallback decision must precede definition-level consumption. Verify + sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed + comparisons, mixed-encoding fallback, and a stable row-shaped hidden-slot placeholder. +- For a filtered scalar page fragment, require one SerDe entry and one batch-level selected-decode + dispatch. Nullable selections must first map logical rows to selected non-NULL physical ranges, + decode those ranges once, and restore NULL slots in place; falling back per NULL run is a review + blocker for a scalar destination that supports in-place expansion. Selection ranges belong to + persistent reader scratch; per-range virtual SerDe/decoder calls in the hot path are a review + blocker. Fixed PLAIN should bulk-gather spans, BYTE_ARRAY PLAIN should scan lengths once, dictionary decode should validate every ID before gathering selected IDs, and stateful encodings should batch-decode/reconstruct and - compact. A NULL-interleaving fallback is acceptable only when it preserves logical output order - without a decoded intermediate column and is counted by - `HybridSelectionNullFallbackBatches`. + compact. Any remaining NULL-interleaving fallback must preserve logical output order without a + decoded intermediate column and be counted by `HybridSelectionNullFallbackBatches`. - Review complex types as a level/shape problem around scalar leaf materialization. Parent offsets, null maps, sibling alignment, page-spanning rows, and child payload counts must remain correct without materializing an intermediate complex column. @@ -271,8 +278,8 @@ format-specific checklist when reviewing Parquet or ORC. file-version key, and assess FileCache, MergeRange, prefetch, requests, and read amplification together. - Require counters for Statistics/Dictionary/Bloom pruning, Page Index selected ranges and skipped - rows/pages, raw and filtered rows, dictionary-row filtering, lazy-read savings, cache sources, and - remote I/O. + rows/pages, raw and filtered rows, dictionary-row filtering, PLAIN direct-predicate batches/rows, + lazy-read savings, cache sources, and remote I/O. - Differential tests must cover absent/invalid statistics, missing or partial Page Index, mixed dictionary/plain encoding, Bloom false positives, NULL/NaN/type conversion, cross-Page batches, nested/repeated columns, multiple Row Groups/Splits, and all/none filtered. diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index 5ad62f04a60532..e703c1d133bbb8 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -369,25 +369,26 @@ Selection inputs are borrowed only for the duration of the decode call. For a flat leaf, the fast path is valid only when the maximum repetition level is zero. It decodes definition-level runs and builds the four-way selection plan in persistent scratch whose capacity -survives adaptive batch changes. When a filtered page fragment contains no definition-level NULL, -the plan is normalized to sorted physical ranges and enters `DataTypeSerDe` and the encoding -decoder once. This is the hybrid selection path: it keeps the dense direct-materialization path, -but moves sparse range traversal inside the concrete decoder instead of repeatedly constructing a -SerDe consumer for every selected run. +survives adaptive batch changes. The plan is normalized to sorted physical non-NULL ranges and +enters `DataTypeSerDe` and the encoding decoder once. If selected NULLs are interleaved, the reader +separately records their logical positions, decodes the compact physical payload, and expands the +final scalar column backwards in place. This is the hybrid selection path: it keeps the dense +direct-materialization path, but moves sparse range traversal inside the concrete decoder instead +of repeatedly constructing a SerDe consumer for every selected or NULL run. The encoding chooses the cheapest inner loop. PLAIN fixed-width values gather ranges with bulk copies; PLAIN BYTE_ARRAY scans every length once and publishes compact source offsets, cumulative -output offsets, and coalesced surviving spans directly to the SerDe; dictionary -encoding decodes and validates the complete ID batch before gathering selected IDs; BOOLEAN, +output offsets, and coalesced surviving spans directly to the SerDe; dictionary encoding validates +IDs in bounded batches and either gathers repeated/literal runs directly or builds one compact +selected-ID batch when a sparse lookup would exceed the L2 cache; BOOLEAN, DELTA, and BYTE_STREAM_SPLIT batch-decode or reconstruct their stateful stream and compact selected values. This follows DuckDB's vector-at-a-time principle while preserving Doris's separate Decoder/SerDe ownership boundary. A filtered non-null value always advances and validates encoding state even when it is not copied, preventing the next batch from decoding shifted values. -If selected NULL slots must be interleaved with values, v2 currently retains the four-run cursor -path so defaults and null-map bits are appended at the exact logical positions without a decoded -intermediate column. `HybridSelectionNullFallbackBatches` makes that conservative path visible; -it is a correctness boundary, not an Arrow fallback. +Scalar destinations with in-place expansion no longer use the per-run NULL fallback. The fallback +remains only for unsupported destination shapes and is visible through +`HybridSelectionNullFallbackBatches`; it is a correctness boundary, not an Arrow fallback. ### 7.2 Page and Encoding Kernel @@ -399,7 +400,7 @@ that parsing step both feed the same level and value-decoder contracts. | Encoding family | Native responsibility | | --- | --- | | PLAIN | Fixed-width little-endian primitives, BOOLEAN bit packing, BYTE_ARRAY lengths, and FIXED_LEN_BYTE_ARRAY widths; identical POD types append by bulk copy, dense fixed-length strings use one byte-span copy plus offset synthesis, sparse fixed-width ranges consume contiguous spans directly, and variable strings reuse decoder-produced offsets without a `StringRef[]` staging pass | -| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, decode and validate the complete ID batch, then gather selected IDs | +| RLE_DICTIONARY / PLAIN_DICTIONARY | Persist dictionary values for the Column Chunk, validate IDs in bounded batches, fuse RLE runs with direct gather for cache-resident dictionaries, and use one compact selected-ID gather for large sparse dictionaries | | RLE / BIT_PACKED levels | Decode definition/repetition levels and preserve runs across page and batch boundaries | | DELTA_BINARY_PACKED | Preserve block/mini-block state, decode one page-fragment batch, and compact selected values in-place | | DELTA_LENGTH_BYTE_ARRAY | Decode lengths and byte payload in lockstep, then retain selected references only | @@ -486,7 +487,10 @@ through decoded dictionary IDs, so dictionary and plain pages have identical beh The typed dictionary is materialized through the logical SerDe once per decoder dictionary generation. Dictionary-entry predicate evaluation, dictionary-ID row filtering, and surviving row-value flattening reuse that same Doris column. A Row Group, file, type, or dictionary-generation -change invalidates it, and every ID is checked before access. +change invalidates it, and every ID is checked before access. Cache-resident or dense reads stream +validated RLE runs directly into the destination column without a page-sized ID vector. Sparse +reads whose dictionary exceeds L2 retain only selected IDs, then perform one gather; malformed late +IDs roll back the destination to its pre-batch size. The persistent leaf reader owns reusable conversion objects, null map, selection ranges, definition/repetition levels, binary references, dictionary state, decompression buffers, and Doris @@ -779,6 +783,17 @@ prefetch to the prefix with a meaningful probability of being reached, and a sus survival ratio estimate may warm output chunks early. This retains correctness because only independently safe conjuncts move and all readers still advance over the same logical batch. +Predicate-only `INT`, `BIGINT`, `FLOAT`, and `DOUBLE` comparisons can bypass Doris-column +materialization when every value page in the Column Chunk is fixed-width PLAIN and the expression +is an exact `=`, `!=`, `<`, `<=`, `>`, or `>=` comparison with a literal. The scheduler compiles +the conjunction into physical-value predicates, the decoder compares selected non-NULL spans in +place, and the reader remaps the compact match bytes to logical nullable rows. NULL comparisons are +false. Mixed encodings, projected predicate columns, casts, logical conversions, compound +expressions, and unsupported types make the decision before either the definition-level or value +cursor advances, so they safely retain ordinary expression evaluation. An exact direct result +leaves only a row-shaped placeholder for the hidden predicate slot and avoids both predicate-column +materialization and later predicate compaction. + Selection state is scheduler-owned across adaptive batches. Its dense filter bitmap is cached by selection generation and batch shape, so every lazily materialized output reader consumes the same bitmap without rebuilding an O(batch-size) array. Logical sizes reset per batch while ordinary @@ -856,6 +871,7 @@ flowchart TD | Dictionary row filter | How often were predicates rewritten, dictionaries read, bitmaps built, and attempts successful or rejected? | | Predicate / raw rows | How many rows were read and rejected, and was lazy materialization worthwhile? | | Predicate compaction | Did selection-first evaluation avoid repeated movement? Inspect `PredicateCompactionTime/Bytes/Count`; single-column rounds retain row mappings and compact at multi-column/delete/output boundaries. | +| PLAIN direct predicate | How many eligible predicate-only physical batches and input rows bypassed Doris-column materialization? Inspect `PlainPredicateDirectBatches/Rows`. | | Avoided projected I/O | How many compressed bytes from projected physical chunks were avoided? `FilteredBytes` deliberately excludes unprojected nested children. | | Metadata lifecycle | How much time was spent reading/parsing the native footer and schema tree, natively reading/parsing page indexes, and evaluating row-group/page-index predicates? | | Parquet Page Cache | What were hit/miss/write counts and compressed/decompressed hit shapes? | From 34c6d81113461601976b808ccdea2ea1d061c344 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 16:30:27 +0800 Subject: [PATCH 31/34] [fix](be) format parquet v2 changes --- be/src/core/data_type_serde/parquet_decode_source.h | 10 ++++------ be/src/format_v2/parquet/native_schema_desc.cpp | 6 ++---- be/src/format_v2/parquet/reader/native/decoder.h | 10 ++++------ be/test/format_v2/parquet/parquet_schema_test.cpp | 7 ++----- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/be/src/core/data_type_serde/parquet_decode_source.h b/be/src/core/data_type_serde/parquet_decode_source.h index aafb112de946f4..e9762385689d07 100644 --- a/be/src/core/data_type_serde/parquet_decode_source.h +++ b/be/src/core/data_type_serde/parquet_decode_source.h @@ -240,8 +240,8 @@ class ParquetDecodeSource { RETURN_IF_ERROR(decode_dictionary_indices(num_values, &indices)); return consumer.consume_indices(indices.data(), indices.size()); } - virtual Status decode_selected_dictionary_values( - const ParquetSelection& selection, ParquetDictionaryValueConsumer& consumer) { + virtual Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) { std::vector indices; RETURN_IF_ERROR(decode_selected_dictionary_indices(selection, &indices)); return consumer.consume_indices(indices.data(), indices.size()); @@ -347,13 +347,11 @@ struct ParquetMaterializationState { return Status::OK(); } - Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, - size_t num_values) { + Status materialize_dictionary(IColumn& column, ParquetDecodeSource& source, size_t num_values) { DORIS_CHECK(typed_dictionary); if (UNLIKELY(dictionary_has_conversion_failures) || source.prefer_dictionary_index_materialization(typed_dictionary->byte_size())) { - dictionary_materialization_strategy = - ParquetDictionaryMaterializationStrategy::INDICES; + dictionary_materialization_strategy = ParquetDictionaryMaterializationStrategy::INDICES; RETURN_IF_ERROR(source.decode_dictionary_indices(num_values, &dictionary_indices)); DORIS_CHECK_EQ(dictionary_indices.size(), num_values); return materialize_dictionary(column); diff --git a/be/src/format_v2/parquet/native_schema_desc.cpp b/be/src/format_v2/parquet/native_schema_desc.cpp index 727b8308ec4b96..1b824a4efc8020 100644 --- a/be/src/format_v2/parquet/native_schema_desc.cpp +++ b/be/src/format_v2/parquet/native_schema_desc.cpp @@ -580,8 +580,7 @@ Status NativeFieldDescriptor::parse_list_field( if (num_children > 0) { const bool structural_wrapper = is_struct_list_node(second_level, first_level.name); const auto& only_child = t_schemas[curr_pos + 2]; - if (num_children == 1 && !structural_wrapper && - has_logical_annotation(second_level)) { + if (num_children == 1 && !structural_wrapper && has_logical_annotation(second_level)) { // The repeated node is already the outer LIST element. Preserve its own LIST/MAP // annotation, but do not interpret its REPEATED marker as another outer array. set_child_node_level(list_field, list_field->definition_level); @@ -592,8 +591,7 @@ Status NativeFieldDescriptor::parse_list_field( } else { RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, list_child)); } - } else if (num_children == 1 && !structural_wrapper && - !is_repeated_node(only_child)) { + } else if (num_children == 1 && !structural_wrapper && !is_repeated_node(only_child)) { // optional field, and the third level element is the nested structure in list // produce nested structure like: LIST, LIST, LIST> // skip bag/list, it's a repeated element. diff --git a/be/src/format_v2/parquet/reader/native/decoder.h b/be/src/format_v2/parquet/reader/native/decoder.h index e3fa2e4fec8358..379176d5e459e0 100644 --- a/be/src/format_v2/parquet/reader/native/decoder.h +++ b/be/src/format_v2/parquet/reader/native/decoder.h @@ -220,9 +220,8 @@ class BaseDictDecoder : public Decoder { return _decode_dictionary_values(num_values, 0, dictionary_size(), consumer); } - Status decode_selected_dictionary_values( - const ParquetSelection& selection, - ParquetDictionaryValueConsumer& consumer) override { + Status decode_selected_dictionary_values(const ParquetSelection& selection, + ParquetDictionaryValueConsumer& consumer) override { const size_t num_dictionary_values = dictionary_size(); size_t cursor = 0; for (const auto& range : selection.ranges) { @@ -311,9 +310,8 @@ class BaseDictDecoder : public Decoder { if (UNLIKELY(literals == 0)) { return Status::IOError("Can't read enough Parquet dictionary indices"); } - const size_t batch = std::min( - {static_cast(literals), num_values - decoded_values, - kLiteralBatchSize}); + const size_t batch = std::min({static_cast(literals), + num_values - decoded_values, kLiteralBatchSize}); _skip_indices.resize(batch); if (UNLIKELY(!_index_batch_decoder->GetLiteralValues(cast_set(batch), _skip_indices.data()))) { diff --git a/be/test/format_v2/parquet/parquet_schema_test.cpp b/be/test/format_v2/parquet/parquet_schema_test.cpp index a832b964ad8690..436674bda035f5 100644 --- a/be/test/format_v2/parquet/parquet_schema_test.cpp +++ b/be/test/format_v2/parquet/parquet_schema_test.cpp @@ -313,8 +313,7 @@ TEST(ParquetSchemaTest, NativeListTupleCompatibilityRequiresEnclosingListName) { EXPECT_EQ(remove_nullable(mismatched_element->data_type)->get_primitive_type(), TYPE_INT); NativeFieldDescriptor matching; - ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}) - .ok()); + ASSERT_TRUE(matching.parse_from_thrift({root(), list("xs"), wrapper("xs_tuple"), item()}).ok()); const auto* matching_element = &matching.get_column(0)->children[0]; EXPECT_EQ(remove_nullable(matching_element->data_type)->get_primitive_type(), TYPE_STRUCT); ASSERT_EQ(matching_element->children.size(), 1); @@ -344,9 +343,7 @@ TEST(ParquetSchemaTest, NativeListPreservesRepeatedAndAnnotatedElementWrappers) repeated_items.__set_type(tparquet::Type::INT32); repeated_items.__set_repetition_type(tparquet::FieldRepetitionType::REPEATED); NativeFieldDescriptor repeated; - ASSERT_TRUE(repeated - .parse_from_thrift( - {root(), outer_list, repeated_wrapper, repeated_items}) + ASSERT_TRUE(repeated.parse_from_thrift({root(), outer_list, repeated_wrapper, repeated_items}) .ok()); const auto* repeated_element = &repeated.get_column(0)->children[0]; EXPECT_EQ(remove_nullable(repeated_element->data_type)->get_primitive_type(), TYPE_STRUCT); From 2524c7836e852d55fca8c2954c3072ddc3ec6970 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 18:09:31 +0800 Subject: [PATCH 32/34] [refactor](be) keep parquet v2 schema and predicate handling native --- be/src/exprs/vectorized_fn_call.cpp | 143 ++++++++++++++++ be/src/exprs/vectorized_fn_call.h | 5 + be/src/exprs/vexpr.h | 13 ++ be/src/format_v2/AGENTS.md | 9 +- be/src/format_v2/parquet/parquet_profile.cpp | 2 - be/src/format_v2/parquet/parquet_profile.h | 2 - be/src/format_v2/parquet/parquet_scan.cpp | 139 +++------------ .../parquet/reader/column_reader.cpp | 2 +- .../format_v2/parquet/reader/column_reader.h | 4 +- .../reader/native/byte_array_plain_decoder.h | 1 - .../reader/native/column_chunk_reader.cpp | 47 +++--- .../reader/native/column_chunk_reader.h | 6 +- .../parquet/reader/native/column_reader.cpp | 159 +++--------------- .../parquet/reader/native/column_reader.h | 33 ++-- .../parquet/reader/native_column_reader.cpp | 28 ++- .../parquet/reader/native_column_reader.h | 7 +- .../parquet/reader/plain_fixed_predicate.h | 125 -------------- .../format_v2/parquet/native_decoder_test.cpp | 133 ++++++++++++--- .../format_v2/parquet/parquet_reader_test.cpp | 2 +- .../format_v2/parquet/parquet_scan_test.cpp | 81 +++++++++ docs/file-scanner-v2-code-review-guide.md | 19 ++- docs/file-scanner-v2-parquet-scan-design.md | 12 +- 22 files changed, 484 insertions(+), 488 deletions(-) delete mode 100644 be/src/format_v2/parquet/reader/plain_fixed_predicate.h diff --git a/be/src/exprs/vectorized_fn_call.cpp b/be/src/exprs/vectorized_fn_call.cpp index b6e5c13d111756..d1bdebe17a2251 100644 --- a/be/src/exprs/vectorized_fn_call.cpp +++ b/be/src/exprs/vectorized_fn_call.cpp @@ -24,9 +24,11 @@ #include #include +#include #include #include +#include "common/compare.h" #include "common/config.h" #include "common/exception.h" #include "common/logging.h" @@ -82,6 +84,82 @@ const static std::set DISTANCE_FUNCS = {L2DistanceApproximate::name const static std::set OPS_FOR_ANN_RANGE_SEARCH = { TExprOpcode::GE, TExprOpcode::LE, TExprOpcode::LE, TExprOpcode::GT, TExprOpcode::LT}; +namespace { + +enum class RawComparisonOp : uint8_t { EQ, NE, LT, LE, GT, GE }; + +std::optional raw_comparison_op(std::string_view function_name, bool reverse) { + RawComparisonOp op; + if (function_name == "eq") { + op = RawComparisonOp::EQ; + } else if (function_name == "ne") { + op = RawComparisonOp::NE; + } else if (function_name == "lt") { + op = RawComparisonOp::LT; + } else if (function_name == "le") { + op = RawComparisonOp::LE; + } else if (function_name == "gt") { + op = RawComparisonOp::GT; + } else if (function_name == "ge") { + op = RawComparisonOp::GE; + } else { + return std::nullopt; + } + if (!reverse || op == RawComparisonOp::EQ || op == RawComparisonOp::NE) { + return op; + } + switch (op) { + case RawComparisonOp::LT: + return RawComparisonOp::GT; + case RawComparisonOp::LE: + return RawComparisonOp::GE; + case RawComparisonOp::GT: + return RawComparisonOp::LT; + case RawComparisonOp::GE: + return RawComparisonOp::LE; + case RawComparisonOp::EQ: + case RawComparisonOp::NE: + break; + } + __builtin_unreachable(); +} + +template +void execute_raw_comparison(const uint8_t* values, size_t num_values, const Field& literal, + RawComparisonOp op, uint8_t* matches) { + const T rhs = literal.get(); + for (size_t row = 0; row < num_values; ++row) { + if (matches[row] == 0) { + continue; + } + const T lhs = unaligned_load(values + row * sizeof(T)); + bool keep = false; + switch (op) { + case RawComparisonOp::EQ: + keep = Compare::equal(lhs, rhs); + break; + case RawComparisonOp::NE: + keep = Compare::not_equal(lhs, rhs); + break; + case RawComparisonOp::LT: + keep = Compare::less(lhs, rhs); + break; + case RawComparisonOp::LE: + keep = Compare::less_equal(lhs, rhs); + break; + case RawComparisonOp::GT: + keep = Compare::greater(lhs, rhs); + break; + case RawComparisonOp::GE: + keep = Compare::greater_equal(lhs, rhs); + break; + } + matches[row] = static_cast(keep); + } +} + +} // namespace + VectorizedFnCall::VectorizedFnCall(const TExprNode& node) : VExpr(node) { _function_name = _fn.name.function_name; } @@ -339,6 +417,71 @@ Status VectorizedFnCall::execute_column_impl(VExprContext* context, const Block* return _do_execute(context, block, selector, count, result_column, nullptr); } +bool VectorizedFnCall::can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + if (data_type == nullptr || !raw_comparison_op(_function_name, false).has_value()) { + return false; + } + auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + if (!slot_literal.has_value() || slot_literal->slot_index != column_id || + slot_literal->literal.is_null()) { + return false; + } + const auto raw_type = remove_nullable(data_type); + if (!remove_nullable(slot_literal->slot_type)->equals(*raw_type) || + !remove_nullable(slot_literal->literal_type)->equals(*raw_type)) { + return false; + } + const auto primitive_type = raw_type->get_primitive_type(); + return primitive_type == TYPE_INT || primitive_type == TYPE_BIGINT || + primitive_type == TYPE_FLOAT || primitive_type == TYPE_DOUBLE; +} + +Status VectorizedFnCall::execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const { + if (!can_execute_on_raw_fixed_values(data_type, column_id)) { + return Status::NotSupported("Expression {} cannot evaluate raw fixed-width values", + expr_name()); + } + DORIS_CHECK(values != nullptr || num_values == 0); + DORIS_CHECK(matches != nullptr || num_values == 0); + const auto slot_literal = expr_zonemap::extract_slot_and_literal(_children); + DORIS_CHECK(slot_literal.has_value()); + const auto op = raw_comparison_op(_function_name, slot_literal->literal_on_left); + DORIS_CHECK(op.has_value()); + const auto primitive_type = remove_nullable(data_type)->get_primitive_type(); + const size_t expected_width = primitive_type == TYPE_INT || primitive_type == TYPE_FLOAT + ? sizeof(uint32_t) + : sizeof(uint64_t); + if (value_width != expected_width) { + return Status::Corruption("Raw expression width {} does not match expected {}", value_width, + expected_width); + } + switch (primitive_type) { + case TYPE_INT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_BIGINT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_FLOAT: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + case TYPE_DOUBLE: + execute_raw_comparison(values, num_values, slot_literal->literal, *op, + matches); + break; + default: + __builtin_unreachable(); + } + return Status::OK(); +} + const std::string& VectorizedFnCall::expr_name() const { return _expr_name; } diff --git a/be/src/exprs/vectorized_fn_call.h b/be/src/exprs/vectorized_fn_call.h index 8343242ce314fd..f4e0dcb8f527fe 100644 --- a/be/src/exprs/vectorized_fn_call.h +++ b/be/src/exprs/vectorized_fn_call.h @@ -57,6 +57,11 @@ class VectorizedFnCall : public VExpr { Status execute_runtime_filter(VExprContext* context, const Block* block, const uint8_t* __restrict filter, size_t count, ColumnPtr& result_column, ColumnPtr* arg_column) const override; + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override; + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr& data_type, int column_id, + uint8_t* matches) const override; Status evaluate_inverted_index(VExprContext* context, uint32_t segment_num_rows) override; ZoneMapFilterResult evaluate_zonemap_filter(const ZoneMapEvalContext& ctx) const override; bool can_evaluate_zonemap_filter() const override; diff --git a/be/src/exprs/vexpr.h b/be/src/exprs/vexpr.h index f54af6f7819acb..77520377af6f93 100644 --- a/be/src/exprs/vexpr.h +++ b/be/src/exprs/vexpr.h @@ -174,6 +174,19 @@ class VExpr { uint8_t* __restrict result_filter_data, size_t rows, bool accept_null, bool* can_filter_all) const; + // Raw fixed-width evaluation is an optional expression capability used before a storage reader + // materializes a column. `matches` is ANDed in place; callers handle NULL rows separately + // because raw value streams contain only non-NULL payloads. + virtual bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const { + return false; + } + virtual Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, + size_t value_width, const DataTypePtr& data_type, + int column_id, uint8_t* matches) const { + return Status::NotSupported("{} cannot evaluate raw fixed-width values", expr_name()); + } + // `is_blockable` means this expr will be blocked in `execute` (e.g. AI Function, Remote Function) [[nodiscard]] virtual bool is_blockable() const { return std::any_of(_children.begin(), _children.end(), diff --git a/be/src/format_v2/AGENTS.md b/be/src/format_v2/AGENTS.md index 3d7fddeec5043e..e417617cf5a64f 100644 --- a/be/src/format_v2/AGENTS.md +++ b/be/src/format_v2/AGENTS.md @@ -137,11 +137,10 @@ instructions as well; this file adds format-v2-specific review expectations. batch or a stateful load-before-build phase. Ordinary predicate/output scans construct `NativeColumnReader`, consume compressed page data through the native decoder, and append directly into the final Doris column. -- Keep logical schema changes distinct from physical decoding. Identical types, integer changes, - FLOAT-to-DOUBLE widening, decimal precision/scale changes, and string-family changes should - materialize through the target SerDe directly. A less common logical cast may use the generic - `ColumnTypeConverter` with a reusable source Doris column, but must not revive - `PhysicalToLogicalConverter` or expose a decoder-owned value batch. +- Keep logical schema changes distinct from physical decoding. The native reader materializes the + projected file type only; `ColumnMapper` and `TableReader` own every file-to-table cast. A type + mismatch at the native reader boundary is an invariant violation, not a reason to create a + reader-local conversion path or expose a decoder-owned value batch. - `CountColumnReader` uses the v2 native `LevelReader`; it selects one representative leaf (the key for MAP) and advances only definition/repetition levels. It exposes no value API and must not be reused as a scan reader or expanded into a fallback path. diff --git a/be/src/format_v2/parquet/parquet_profile.cpp b/be/src/format_v2/parquet/parquet_profile.cpp index c5608bcaaa8f08..9079b5f6060eb0 100644 --- a/be/src/format_v2/parquet/parquet_profile.cpp +++ b/be/src/format_v2/parquet/parquet_profile.cpp @@ -197,7 +197,6 @@ void ParquetProfile::init(RuntimeProfile* profile) { TUnit::UNIT, parquet_profile, 1); rows_filtered_by_dict_filter = ADD_CHILD_COUNTER_WITH_LEVEL(profile, "RowsFilteredByDictFilter", TUnit::UNIT, parquet_profile, 1); - convert_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "ConvertTime", parquet_profile, 1); bloom_filter_read_time = ADD_CHILD_TIMER_WITH_LEVEL(profile, "BloomFilterReadTime", parquet_profile, 1); } @@ -282,7 +281,6 @@ ParquetColumnReaderProfile ParquetProfile::column_reader_profile() const { .decode_dictionary_time = decode_dict_time, .decode_level_time = decode_level_time, .decode_null_map_time = decode_null_map_time, - .convert_time = convert_time, .page_index_read_calls = page_index_read_calls, .skip_page_header_count = skip_page_header_num, .parse_page_header_count = parse_page_header_num, diff --git a/be/src/format_v2/parquet/parquet_profile.h b/be/src/format_v2/parquet/parquet_profile.h index 60289208188922..f1b06be19d60c4 100644 --- a/be/src/format_v2/parquet/parquet_profile.h +++ b/be/src/format_v2/parquet/parquet_profile.h @@ -50,7 +50,6 @@ struct ParquetColumnReaderProfile { RuntimeProfile::Counter* decode_dictionary_time = nullptr; RuntimeProfile::Counter* decode_level_time = nullptr; RuntimeProfile::Counter* decode_null_map_time = nullptr; - RuntimeProfile::Counter* convert_time = nullptr; RuntimeProfile::Counter* page_index_read_calls = nullptr; RuntimeProfile::Counter* skip_page_header_count = nullptr; RuntimeProfile::Counter* parse_page_header_count = nullptr; @@ -215,7 +214,6 @@ struct ParquetProfile { RuntimeProfile::Counter* dict_filter_unsupported_columns = nullptr; RuntimeProfile::Counter* dict_filter_read_failures = nullptr; RuntimeProfile::Counter* rows_filtered_by_dict_filter = nullptr; - RuntimeProfile::Counter* convert_time = nullptr; RuntimeProfile::Counter* bloom_filter_read_time = nullptr; }; diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 3b94371812dea0..981541c37732c3 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -16,12 +16,12 @@ #include "format_v2/parquet/parquet_scan.h" #include +#include #include #include #include #include #include -#include #include #include @@ -31,10 +31,7 @@ #include "core/block/block.h" #include "core/column/column_vector.h" #include "exprs/vcompound_pred.h" -#include "exprs/vectorized_fn_call.h" #include "exprs/vexpr_context.h" -#include "exprs/vliteral.h" -#include "exprs/vslot_ref.h" #include "format_v2/parquet/parquet_column_schema.h" #include "format_v2/parquet/parquet_file_context.h" #include "format_v2/parquet/parquet_statistics.h" @@ -101,108 +98,6 @@ bool should_sample_adaptive_predicate(size_t samples, size_t batch_sequence) { namespace { -std::optional plain_predicate_op(std::string_view function_name, - bool reverse) { - PlainFixedPredicateOp op; - if (function_name == "eq") { - op = PlainFixedPredicateOp::EQ; - } else if (function_name == "ne") { - op = PlainFixedPredicateOp::NE; - } else if (function_name == "lt") { - op = PlainFixedPredicateOp::LT; - } else if (function_name == "le") { - op = PlainFixedPredicateOp::LE; - } else if (function_name == "gt") { - op = PlainFixedPredicateOp::GT; - } else if (function_name == "ge") { - op = PlainFixedPredicateOp::GE; - } else { - return std::nullopt; - } - if (!reverse) { - return op; - } - switch (op) { - case PlainFixedPredicateOp::LT: - return PlainFixedPredicateOp::GT; - case PlainFixedPredicateOp::LE: - return PlainFixedPredicateOp::GE; - case PlainFixedPredicateOp::GT: - return PlainFixedPredicateOp::LT; - case PlainFixedPredicateOp::GE: - return PlainFixedPredicateOp::LE; - case PlainFixedPredicateOp::EQ: - case PlainFixedPredicateOp::NE: - return op; - } - __builtin_unreachable(); -} - -std::optional> compile_plain_fixed_predicates( - const VExprContextSPtrs& conjuncts, const DataTypePtr& column_type, size_t block_position) { - const auto primitive_type = remove_nullable(column_type)->get_primitive_type(); - if (primitive_type != TYPE_INT && primitive_type != TYPE_BIGINT && - primitive_type != TYPE_FLOAT && primitive_type != TYPE_DOUBLE) { - return std::nullopt; - } - std::vector predicates; - predicates.reserve(conjuncts.size()); - for (const auto& conjunct : conjuncts) { - const auto* function = - conjunct == nullptr ? nullptr - : dynamic_cast(conjunct->root().get()); - if (function == nullptr || function->children().size() != 2) { - return std::nullopt; - } - const auto* left_slot = dynamic_cast(function->children()[0].get()); - const auto* right_slot = dynamic_cast(function->children()[1].get()); - const auto* left_literal = dynamic_cast(function->children()[0].get()); - const auto* right_literal = dynamic_cast(function->children()[1].get()); - const bool reverse = right_slot != nullptr && left_literal != nullptr; - const auto* slot = reverse ? right_slot : left_slot; - const auto* literal = reverse ? left_literal : right_literal; - if (slot == nullptr || literal == nullptr || - static_cast(slot->column_id()) != block_position) { - return std::nullopt; - } - if (!remove_nullable(literal->get_data_type())->equals(*remove_nullable(column_type))) { - return std::nullopt; - } - const auto op = plain_predicate_op(function->function_name(), reverse); - if (!op.has_value()) { - return std::nullopt; - } - Field value; - literal->get_column_ptr()->get(0, value); - if (value.is_null()) { - return std::nullopt; - } - switch (primitive_type) { - case TYPE_INT: - predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::INT32, *op, - value.get())); - break; - case TYPE_BIGINT: - predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::INT64, *op, - value.get())); - break; - case TYPE_FLOAT: - predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::FLOAT, *op, - value.get())); - break; - case TYPE_DOUBLE: - predicates.push_back(PlainFixedPredicate::create(PlainFixedPredicateType::DOUBLE, *op, - value.get())); - break; - default: - __builtin_unreachable(); - } - } - return predicates.empty() - ? std::nullopt - : std::optional>(std::move(predicates)); -} - detail::PredicateConjunctSchedule build_predicate_conjunct_schedule( const format::FileScanRequest& request); @@ -1522,6 +1417,20 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, selection->resize(static_cast(batch_rows)); } const auto& schedule = predicate_conjunct_schedule(request); + std::unordered_set residual_predicate_positions; + auto remember_residual_positions = [&](const VExprContextSPtrs& conjuncts) { + for (const auto& conjunct : conjuncts) { + std::set positions; + conjunct->root()->collect_slot_column_ids(positions); + for (const int position : positions) { + if (position >= 0) { + residual_predicate_positions.insert(cast_set(position)); + } + } + } + }; + remember_residual_positions(schedule.remaining_conjuncts); + remember_residual_positions(request.delete_conjuncts); const size_t predicate_batch_sequence = _predicate_batch_sequence++; const bool can_read_predicate_columns_round_by_round = !schedule.single_column_conjuncts.empty(); @@ -1664,17 +1573,19 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, } if (single_column_conjuncts != nullptr && + !residual_predicate_positions.contains(block_position) && request.is_predicate_only(format::LocalColumnId(cast_set(local_id)))) { - auto predicates = compile_plain_fixed_predicates( - *single_column_conjuncts, file_block->get_by_position(block_position).type, - block_position); - if (predicates.has_value()) { + VExprSPtrs direct_conjuncts; + direct_conjuncts.reserve(single_column_conjuncts->size()); + std::ranges::transform(*single_column_conjuncts, std::back_inserter(direct_conjuncts), + [](const auto& context) { return context->root(); }); + if (!direct_conjuncts.empty()) { const uint16_t selected_rows_before = *selected_rows; IColumn::Filter compact_filter; bool used_filter = false; RETURN_IF_ERROR(column_reader->select_with_plain_filter( - *selection, *selected_rows, batch_rows, *predicates, &compact_filter, - &used_filter)); + *selection, *selected_rows, batch_rows, direct_conjuncts, + cast_set(block_position), &compact_filter, &used_filter)); if (used_filter) { DORIS_CHECK_EQ(compact_filter.size(), selected_rows_before); update_counter_if_not_null(_scan_profile.plain_predicate_direct_batches, 1); @@ -1690,8 +1601,8 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, *selected_rows = apply_compact_filter_to_selection( compact_filter, selection, selected_rows_before); } - // Predicate-only values are dead after this exact comparison. Preserve only a - // row-shaped placeholder so later block positions keep their stable identity. + // This slot is absent from every residual/delete conjunct, so no later + // expression can observe its payload. Keep only the block row-shape contract. auto placeholder = column->clone_empty(); placeholder->insert_many_defaults(*selected_rows); file_block->replace_by_position(block_position, std::move(placeholder)); diff --git a/be/src/format_v2/parquet/reader/column_reader.cpp b/be/src/format_v2/parquet/reader/column_reader.cpp index 2883dcd4f36c88..f23559e5dbf9f0 100644 --- a/be/src/format_v2/parquet/reader/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/column_reader.cpp @@ -82,7 +82,7 @@ Status ParquetColumnReader::select_with_dictionary_filter(const SelectionVector& } Status ParquetColumnReader::select_with_plain_filter(const SelectionVector&, uint16_t, int64_t, - const std::vector&, + const VExprSPtrs&, int, IColumn::Filter* row_filter, bool* used_filter) { DORIS_CHECK(row_filter != nullptr); diff --git a/be/src/format_v2/parquet/reader/column_reader.h b/be/src/format_v2/parquet/reader/column_reader.h index 59c0f0f63f33c7..bd4b73e3aa5172 100644 --- a/be/src/format_v2/parquet/reader/column_reader.h +++ b/be/src/format_v2/parquet/reader/column_reader.h @@ -21,8 +21,8 @@ #include "common/status.h" #include "core/column/column.h" #include "core/data_type/data_type.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/parquet_profile.h" -#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "format_v2/parquet/selection_vector.h" namespace doris::format::parquet { @@ -65,7 +65,7 @@ class ParquetColumnReader { // predicate column. Implementations must leave the cursor untouched when used_filter=false. virtual Status select_with_plain_filter(const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, - const std::vector& predicates, + const VExprSPtrs& conjuncts, int column_id, IColumn::Filter* row_filter, bool* used_filter); // Native statistics are cumulative and can be recursively aggregated for complex columns. diff --git a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h index 435688f485b217..45bc82fe823f1a 100644 --- a/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h +++ b/be/src/format_v2/parquet/reader/native/byte_array_plain_decoder.h @@ -27,7 +27,6 @@ #include "common/status.h" #include "core/data_type/data_type.h" #include "core/types.h" -#include "format/format_common.h" #include "format_v2/parquet/reader/native/decoder.h" #include "util/bit_util.h" #include "util/coding.h" diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index 2f508ca0aa3b85..fb4fdde26a48d3 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -38,6 +38,7 @@ #include "core/custom_allocator.h" #include "core/data_type_serde/data_type_serde.h" #include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vexpr.h" #include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" @@ -615,24 +616,30 @@ Status decode_selected_nullable_values(IColumn& column, const DataTypeSerDe& ser class PlainPredicateConsumer final : public ParquetFixedValueConsumer { public: - PlainPredicateConsumer(const std::vector& predicates, + PlainPredicateConsumer(const VExprSPtrs& conjuncts, DataTypePtr data_type, int column_id, IColumn::Filter* matches) - : _predicates(predicates), _matches(matches) { + : _conjuncts(conjuncts), + _data_type(std::move(data_type)), + _column_id(column_id), + _matches(matches) { DORIS_CHECK(_matches != nullptr); } Status consume(const uint8_t* values, size_t num_values, size_t value_width) override { const size_t old_size = _matches->size(); _matches->resize_fill(old_size + num_values, 1); - for (const auto& predicate : _predicates) { - RETURN_IF_ERROR(predicate.evaluate(values, num_values, value_width, - _matches->data() + old_size)); + for (const auto& conjunct : _conjuncts) { + RETURN_IF_ERROR(conjunct->execute_on_raw_fixed_values(values, num_values, value_width, + _data_type, _column_id, + _matches->data() + old_size)); } return Status::OK(); } private: - const std::vector& _predicates; + const VExprSPtrs& _conjuncts; + DataTypePtr _data_type; + int _column_id; IColumn::Filter* _matches; }; @@ -1338,32 +1345,21 @@ Status ColumnChunkReader::materialize_values( template bool ColumnChunkReader::can_filter_plain_values( - const std::vector& predicates) const { - if (predicates.empty() || _current_encoding != tparquet::Encoding::PLAIN || + const VExprSPtrs& conjuncts, int column_id) const { + if (conjuncts.empty() || _current_encoding != tparquet::Encoding::PLAIN || (_metadata.type != tparquet::Type::INT32 && _metadata.type != tparquet::Type::INT64 && _metadata.type != tparquet::Type::FLOAT && _metadata.type != tparquet::Type::DOUBLE)) { return false; } - const size_t physical_width = - _metadata.type == tparquet::Type::INT32 || _metadata.type == tparquet::Type::FLOAT - ? sizeof(uint32_t) - : sizeof(uint64_t); - return std::ranges::all_of(predicates, [&](const auto& predicate) { - const bool matching_type = (_metadata.type == tparquet::Type::INT32 && - predicate.type() == PlainFixedPredicateType::INT32) || - (_metadata.type == tparquet::Type::INT64 && - predicate.type() == PlainFixedPredicateType::INT64) || - (_metadata.type == tparquet::Type::FLOAT && - predicate.type() == PlainFixedPredicateType::FLOAT) || - (_metadata.type == tparquet::Type::DOUBLE && - predicate.type() == PlainFixedPredicateType::DOUBLE); - return matching_type && predicate.value_width() == physical_width; + return std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); }); } template Status ColumnChunkReader::filter_plain_values( - const std::vector& predicates, ColumnSelectVector& select_vector, + const VExprSPtrs& conjuncts, int column_id, ColumnSelectVector& select_vector, NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn::Filter* row_filter, bool* used_filter) { DORIS_CHECK(selected_nulls != nullptr); @@ -1372,7 +1368,7 @@ Status ColumnChunkReader::filter_plain_values( DORIS_CHECK(used_filter != nullptr); *used_filter = false; row_filter->clear(); - if (!can_filter_plain_values(predicates)) { + if (!can_filter_plain_values(conjuncts, column_id)) { return Status::OK(); } if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { @@ -1424,7 +1420,8 @@ Status ColumnChunkReader::filter_plain_values( if (selection.selected_values == 0) { RETURN_IF_ERROR(_page_decoder->skip_values(selection.total_values)); } else { - PlainPredicateConsumer consumer(predicates, physical_matches); + PlainPredicateConsumer consumer(conjuncts, _field_schema->data_type, column_id, + physical_matches); RETURN_IF_ERROR(_page_decoder->decode_selected_fixed_values(selection, consumer)); DORIS_CHECK_EQ(physical_matches->size(), selection.selected_values); } diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 6a4b61b3e8b154..807a2531d2cfe9 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -30,12 +30,12 @@ #include "core/column/column_string.h" #include "core/data_type/data_type.h" #include "core/data_type_serde/parquet_decode_source.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/common.h" #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" -#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "util/slice.h" namespace doris { @@ -165,11 +165,11 @@ class ColumnChunkReader { // Evaluate selected fixed-width PLAIN values and return one keep byte per selected logical // row. NULL comparisons are false and therefore never enter the physical consumer. - Status filter_plain_values(const std::vector& predicates, + Status filter_plain_values(const VExprSPtrs& conjuncts, int column_id, ColumnSelectVector& select_vector, NullMap* selected_nulls, IColumn::Filter* physical_matches, IColumn::Filter* row_filter, bool* used_filter); - bool can_filter_plain_values(const std::vector& predicates) const; + bool can_filter_plain_values(const VExprSPtrs& conjuncts, int column_id) const; // Get the repetition level decoder of current page. LevelDecoder& rep_level_decoder() { return _rep_level_decoder; } diff --git a/be/src/format_v2/parquet/reader/native/column_reader.cpp b/be/src/format_v2/parquet/reader/native/column_reader.cpp index 3cfa8fead56f5a..aa22e4f5f7098d 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_reader.cpp @@ -38,6 +38,7 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_struct.h" #include "core/data_type/define_primitive_type.h" +#include "exprs/vexpr.h" #include "format_v2/parquet/native_schema_desc.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native/level_decoder.h" @@ -60,32 +61,6 @@ ParquetTimeUnit parquet_time_unit(const tparquet::TimeUnit& unit) { return ParquetTimeUnit::UNKNOWN; } -bool is_direct_integer_type(PrimitiveType type) { - switch (type) { - case TYPE_TINYINT: - case TYPE_SMALLINT: - case TYPE_INT: - case TYPE_BIGINT: - case TYPE_LARGEINT: - return true; - default: - return false; - } -} - -bool is_direct_decimal_type(PrimitiveType type) { - switch (type) { - case TYPE_DECIMALV2: - case TYPE_DECIMAL32: - case TYPE_DECIMAL64: - case TYPE_DECIMAL128I: - case TYPE_DECIMAL256: - return true; - default: - return false; - } -} - template bool release_vector_if_oversized(std::vector* values, size_t max_retained_bytes) { DORIS_CHECK(values != nullptr); @@ -100,10 +75,6 @@ size_t retained_set_bytes(const std::unordered_set& values) { return values.bucket_count() * sizeof(void*) + values.size() * sizeof(size_t); } -bool is_direct_binary_type(PrimitiveType type) { - return is_string_type(type) || type == TYPE_VARBINARY; -} - Status validate_decimal_physical_type(const NativeFieldSchema& field, int precision, int scale) { if (precision <= 0 || scale < 0 || scale > precision) { return Status::Corruption("Parquet decimal field {} has invalid precision {} and scale {}", @@ -296,23 +267,6 @@ void mark_local_timestamp_defaults(const NativeFieldSchema& field, const DataTyp } } -// The target SerDe can fuse physical decode with these logical type changes. Less common schema -// changes retain the generic file-format converter as a compatibility path: the decoder still -// exposes raw spans, but the source SerDe first materializes a reusable source column before the -// generic logical cast. Ordinary scans and numeric widening never allocate that source column. -bool serde_can_materialize_directly(const DataTypePtr& source_type, - const DataTypePtr& target_type) { - const auto source = remove_nullable(source_type)->get_primitive_type(); - const auto target = remove_nullable(target_type)->get_primitive_type(); - return source == target || (is_direct_integer_type(source) && is_direct_integer_type(target)) || - (source == TYPE_FLOAT && target == TYPE_DOUBLE) || - (is_direct_decimal_type(source) && is_direct_decimal_type(target)) || - // Parquet STRING and VARBINARY share BYTE_ARRAY bytes. Materializing through the target - // SerDe preserves those bytes and avoids a converter whose scratch column uses the - // String representation instead of the native ColumnVarbinary representation. - (is_direct_binary_type(source) && is_direct_binary_type(target)); -} - Status init_decode_context(const NativeFieldSchema& field, const cctz::time_zone* ctz, ParquetDecodeContext* context) { DORIS_CHECK(context != nullptr); @@ -797,12 +751,6 @@ void ScalarColumnReader::release_batch_scratch( if (release_selection) { _select_vector = ColumnSelectVector(); } - if (_logical_conversion_scratch_bytes > max_retained_bytes) { - _logical_converter.reset(); - _converter_source_type = nullptr; - _converter_target_type = nullptr; - _logical_conversion_scratch_bytes = 0; - } } template @@ -1100,8 +1048,8 @@ Status ScalarColumnReader::_read_nested_column( template Status ScalarColumnReader::_read_plain_filter_values( - size_t num_values, const std::vector& predicates, - FilterMap& filter_map, IColumn::Filter* row_filter) { + size_t num_values, const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + IColumn::Filter* row_filter) { DORIS_CHECK(row_filter != nullptr); _null_run_lengths.clear(); if (_chunk_reader->max_def_level() > 0) { @@ -1143,8 +1091,8 @@ Status ScalarColumnReader::_read_plain_filter_value _filter_map_index += num_values; bool used_filter = false; RETURN_IF_ERROR(_chunk_reader->filter_plain_values( - predicates, _select_vector, &_plain_predicate_nulls, &_plain_predicate_matches, - row_filter, &used_filter)); + conjuncts, column_id, _select_vector, &_plain_predicate_nulls, + &_plain_predicate_matches, row_filter, &used_filter)); // Chunk encodings are prevalidated before any definition level is consumed, so a false result // here would make a materializing fallback observe an advanced level cursor. DORIS_CHECK(used_filter); @@ -1153,9 +1101,8 @@ Status ScalarColumnReader::_read_plain_filter_value template Status ScalarColumnReader::read_plain_filter( - const std::vector& predicates, FilterMap& filter_map, - size_t batch_size, IColumn::Filter* row_filter, size_t* read_rows, bool* eof, - bool* used_filter) { + const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, size_t batch_size, + IColumn::Filter* row_filter, size_t* read_rows, bool* eof, bool* used_filter) { DORIS_CHECK(row_filter != nullptr); DORIS_CHECK(read_rows != nullptr); DORIS_CHECK(eof != nullptr); @@ -1163,22 +1110,13 @@ Status ScalarColumnReader::read_plain_filter( row_filter->clear(); *read_rows = 0; *used_filter = false; - if (_in_nested || predicates.empty()) { + if (_in_nested || conjuncts.empty()) { return Status::OK(); } - const auto source_primitive = remove_nullable(_field_schema->data_type)->get_primitive_type(); - const bool matching_source_type = - std::ranges::all_of(predicates, [&](const PlainFixedPredicate& predicate) { - return (source_primitive == TYPE_INT && - predicate.type() == PlainFixedPredicateType::INT32) || - (source_primitive == TYPE_BIGINT && - predicate.type() == PlainFixedPredicateType::INT64) || - (source_primitive == TYPE_FLOAT && - predicate.type() == PlainFixedPredicateType::FLOAT) || - (source_primitive == TYPE_DOUBLE && - predicate.type() == PlainFixedPredicateType::DOUBLE); - }); - if (!matching_source_type) { + if (!std::ranges::all_of(conjuncts, [&](const auto& conjunct) { + return conjunct != nullptr && + conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); + })) { return Status::OK(); } // Levels use RLE/BIT_PACKED, while every value page must be PLAIN. Reject mixed-encoding @@ -1207,7 +1145,7 @@ Status ScalarColumnReader::read_plain_filter( } else { RETURN_IF_ERROR(_chunk_reader->parse_page_header()); RETURN_IF_ERROR(_chunk_reader->load_page_data_idempotent()); - if (!_chunk_reader->can_filter_plain_values(predicates)) { + if (!_chunk_reader->can_filter_plain_values(conjuncts, column_id)) { return Status::OK(); } size_t has_read = 0; @@ -1219,8 +1157,8 @@ Status ScalarColumnReader::read_plain_filter( const size_t values = std::min(static_cast(range.to() - range.from()), batch_size - has_read); IColumn::Filter fragment_filter; - RETURN_IF_ERROR( - _read_plain_filter_values(values, predicates, filter_map, &fragment_filter)); + RETURN_IF_ERROR(_read_plain_filter_values(values, conjuncts, column_id, filter_map, + &fragment_filter)); row_filter->insert(row_filter->end(), fragment_filter.begin(), fragment_filter.end()); has_read += values; *read_rows += values; @@ -1397,34 +1335,17 @@ Status ScalarColumnReader::read_column_data( const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size) { - const DataTypePtr target_type = remove_nullable(type); - const DataTypePtr source_type = remove_nullable(_field_schema->data_type); - const bool needs_logical_conversion = - !is_dict_filter && !serde_can_materialize_directly(source_type, target_type); - - ColumnPtr converted_source_column; - ColumnPtr* read_column = &doris_column; - DataTypePtr materialization_type = target_type; - if (needs_logical_conversion) { - if (_logical_converter == nullptr || _converter_source_type != source_type.get() || - _converter_target_type != target_type.get()) { - _logical_converter = converter::ColumnTypeConverter::get_converter( - source_type, target_type, converter::FileFormat::PARQUET); - if (!_logical_converter->support()) { - return Status::InternalError( - "The column type of '{}' has changed and is not supported: {}", - _field_schema->name, _logical_converter->get_error_msg()); - } - _converter_source_type = source_type.get(); - _converter_target_type = target_type.get(); - } - converted_source_column = _logical_converter->get_column(source_type, doris_column, type); - read_column = &converted_source_column; - materialization_type = remove_nullable(_logical_converter->get_type()); + const DataTypePtr materialization_type = remove_nullable(type); + const DataTypePtr file_type = remove_nullable(_field_schema->data_type); + if (!is_dict_filter && !file_type->equals(*materialization_type)) { + // File-to-table casts belong to ColumnMapper. Reaching this reader with a table type would + // also evaluate file-local predicates against values produced under the wrong type. + return Status::InternalError( + "Native Parquet reader type mismatch for file column '{}': file={}, requested={}", + _field_schema->name, file_type->get_name(), materialization_type->get_name()); } - const DataTypePtr serde_type = - is_dict_filter ? remove_nullable(_field_schema->data_type) : materialization_type; + const DataTypePtr serde_type = is_dict_filter ? file_type : materialization_type; if (_serde_type != serde_type.get() || _dictionary_index_only != is_dict_filter) { _serde_type = serde_type.get(); _serde = serde_type->get_serde(); @@ -1433,37 +1354,13 @@ Status ScalarColumnReader::read_column_data( } _decode_context.dictionary_index_only = is_dict_filter; - auto finish_logical_conversion = [&]() -> Status { - if (!needs_logical_conversion) { - return Status::OK(); - } - DORIS_CHECK(_logical_converter != nullptr); - doris_column = IColumn::mutate(std::move(doris_column)); - auto converted_column = doris_column->assert_mutable(); - if (is_column_nullable(*converted_column)) { - const auto* source_nullable = - check_and_get_column(*converted_source_column); - DORIS_CHECK(source_nullable != nullptr); - auto& destination_null_map = - assert_cast(*converted_column).get_null_map_data(); - const auto& source_null_map = source_nullable->get_null_map_data(); - destination_null_map.insert(source_null_map.begin(), source_null_map.end()); - } - SCOPED_RAW_TIMER(&_convert_time); - RETURN_IF_ERROR(_logical_converter->convert(converted_source_column, converted_column)); - _logical_conversion_scratch_bytes = converted_source_column->allocated_bytes(); - doris_column = std::move(converted_column); - return Status::OK(); - }; - _def_levels.clear(); _rep_levels.clear(); *read_rows = 0; if (_in_nested) { - RETURN_IF_ERROR(_read_nested_column(*read_column, materialization_type, filter_map, - batch_size, read_rows, eof, is_dict_filter)); - return finish_logical_conversion(); + return _read_nested_column(doris_column, materialization_type, filter_map, batch_size, + read_rows, eof, is_dict_filter); } int64_t right_row = 0; @@ -1518,7 +1415,7 @@ Status ScalarColumnReader::read_column_data( if (skip_whole_batch) { RETURN_IF_ERROR(_skip_values(read_values)); } else { - RETURN_IF_ERROR(_read_values(read_values, *read_column, materialization_type, + RETURN_IF_ERROR(_read_values(read_values, doris_column, materialization_type, filter_map, is_dict_filter)); } has_read += read_values; @@ -1539,7 +1436,7 @@ Status ScalarColumnReader::read_column_data( } } - return finish_logical_conversion(); + return Status::OK(); } Status ArrayColumnReader::init(std::unique_ptr element_reader, diff --git a/be/src/format_v2/parquet/reader/native/column_reader.h b/be/src/format_v2/parquet/reader/native/column_reader.h index 98be7df08c8b65..a4f0a56c8efff6 100644 --- a/be/src/format_v2/parquet/reader/native/column_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_reader.h @@ -29,14 +29,13 @@ #include "common/status.h" #include "core/data_type/data_type.h" -#include "format/column_type_convert.h" -#include "format/generic_reader.h" +#include "exprs/vexpr_fwd.h" #include "format_v2/parquet/native_schema_node.h" #include "format_v2/parquet/reader/native/column_chunk_reader.h" #include "format_v2/parquet/reader/native/common.h" -#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "io/fs/buffered_reader.h" #include "io/fs/file_reader_writer_fwd.h" +#include "storage/segment/row_ranges.h" namespace cctz { class time_zone; @@ -48,6 +47,8 @@ struct IOContext; namespace doris::format::parquet::native { using ::doris::ColumnString; +using segment_v2::RowRange; +using segment_v2::RowRanges; #ifdef BE_TEST Status init_decode_context_for_test(const NativeFieldSchema& field, const cctz::time_zone* ctz, @@ -77,7 +78,6 @@ class ColumnReader { decode_dict_time(0), decode_level_time(0), decode_null_map_time(0), - convert_time(0), skip_page_header_num(0), parse_page_header_num(0), read_page_header_time(0), @@ -90,8 +90,7 @@ class ColumnReader { page_cache_compressed_hit_counter(0), page_cache_decompressed_hit_counter(0) {} - ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time, - int64_t convert_time_) + ColumnStatistics(ColumnChunkReaderStatistics& cs, int64_t null_map_time) : page_index_read_calls(0), decompress_time(cs.decompress_time), decompress_cnt(cs.decompress_cnt), @@ -104,7 +103,6 @@ class ColumnReader { decode_dict_time(cs.decode_dict_time), decode_level_time(cs.decode_level_time), decode_null_map_time(null_map_time), - convert_time(convert_time_), skip_page_header_num(cs.skip_page_header_num), parse_page_header_num(cs.parse_page_header_num), read_page_header_time(cs.read_page_header_time), @@ -130,7 +128,6 @@ class ColumnReader { int64_t decode_dict_time; int64_t decode_level_time; int64_t decode_null_map_time; - int64_t convert_time; int64_t skip_page_header_num; int64_t parse_page_header_num; int64_t read_page_header_time; @@ -159,7 +156,6 @@ class ColumnReader { decode_dict_time += col_statistics.decode_dict_time; decode_level_time += col_statistics.decode_level_time; decode_null_map_time += col_statistics.decode_null_map_time; - convert_time += col_statistics.convert_time; skip_page_header_num += col_statistics.skip_page_header_num; parse_page_header_num += col_statistics.parse_page_header_num; read_page_header_time += col_statistics.read_page_header_time; @@ -192,7 +188,7 @@ class ColumnReader { // Evaluate a predicate-only scalar directly from fixed-width PLAIN page bytes. The default is // a non-consuming fallback for nested and synthetic readers. - virtual Status read_plain_filter(const std::vector&, FilterMap&, size_t, + virtual Status read_plain_filter(const VExprSPtrs&, int, FilterMap&, size_t, IColumn::Filter* row_filter, size_t* read_rows, bool* eof, bool* used_filter) { DORIS_CHECK(row_filter != nullptr); @@ -283,9 +279,9 @@ class ScalarColumnReader : public ColumnReader { const std::shared_ptr& root_node, FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof, bool is_dict_filter, int64_t real_column_size = -1) override; - Status read_plain_filter(const std::vector& predicates, - FilterMap& filter_map, size_t batch_size, IColumn::Filter* row_filter, - size_t* read_rows, bool* eof, bool* used_filter) override; + Status read_plain_filter(const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, + size_t batch_size, IColumn::Filter* row_filter, size_t* read_rows, + bool* eof, bool* used_filter) override; Status read_column_levels(FilterMap& filter_map, size_t batch_size, size_t* read_rows, bool* eof) override; Result convert_dict_column_to_string_column( @@ -294,8 +290,7 @@ class ScalarColumnReader : public ColumnReader { const std::vector& get_rep_level() const override { return _rep_levels; } const std::vector& get_def_level() const override { return _def_levels; } ColumnStatistics column_statistics() override { - return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time, - _convert_time); + return ColumnStatistics(_chunk_reader->chunk_statistics(), _decode_null_map_time); } void close() override {} @@ -383,9 +378,6 @@ class ScalarColumnReader : public ColumnReader { DataTypeSerDeSPtr _serde; const IDataType* _serde_type = nullptr; - std::unique_ptr _logical_converter; - const IDataType* _converter_source_type = nullptr; - const IDataType* _converter_target_type = nullptr; ParquetDecodeContext _decode_context; ParquetMaterializationState _materialization_state; bool _dictionary_index_only = false; @@ -398,8 +390,6 @@ class ScalarColumnReader : public ColumnReader { IColumn::Filter _plain_predicate_matches; FilterMap _nested_filter_map; ColumnSelectVector _select_vector; - int64_t _convert_time = 0; - size_t _logical_conversion_scratch_bytes = 0; uint8_t _oversized_scratch_idle_batches = 0; size_t retained_batch_scratch_bytes() const; @@ -408,8 +398,7 @@ class ScalarColumnReader : public ColumnReader { Status _skip_values(size_t num_values); Status _read_values(size_t num_values, ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, bool is_dict_filter); - Status _read_plain_filter_values(size_t num_values, - const std::vector& predicates, + Status _read_plain_filter_values(size_t num_values, const VExprSPtrs& conjuncts, int column_id, FilterMap& filter_map, IColumn::Filter* row_filter); Status _read_nested_column(ColumnPtr& doris_column, const DataTypePtr& type, FilterMap& filter_map, size_t batch_size, size_t* read_rows, diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 64506ccf604eaf..355dd79f5422e2 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -239,7 +239,7 @@ Status NativeColumnReader::init( DORIS_CHECK(range.start >= 0); DORIS_CHECK(range.length > 0); DORIS_CHECK(range.start + range.length <= _row_group_rows); - _row_ranges.add(::doris::RowRange(range.start, range.start + range.length)); + _row_ranges.add(segment_v2::RowRange(range.start, range.start + range.length)); } // Offset indexes are immutable row-group metadata owned by ParquetScanScheduler. Sharing them // avoids retaining the full N-column page-location map once per projected reader. @@ -337,10 +337,10 @@ Status NativeColumnReader::read_with_filter(int64_t rows, const uint8_t* filter_ return Status::OK(); } -Status NativeColumnReader::read_with_plain_filter( - int64_t rows, const uint8_t* filter_data, bool filter_all, - const std::vector& predicates, IColumn::Filter* row_filter, - int64_t* rows_read, bool* used_filter) { +Status NativeColumnReader::read_with_plain_filter(int64_t rows, const uint8_t* filter_data, + bool filter_all, const VExprSPtrs& conjuncts, + int column_id, IColumn::Filter* row_filter, + int64_t* rows_read, bool* used_filter) { DORIS_CHECK(rows >= 0); DORIS_CHECK(row_filter != nullptr); DORIS_CHECK(rows_read != nullptr); @@ -362,7 +362,7 @@ Status NativeColumnReader::read_with_plain_filter( IColumn::Filter loop_filter; bool loop_used = false; RETURN_IF_ERROR(_native_reader->read_plain_filter( - predicates, filter, static_cast(rows - *rows_read), &loop_filter, + conjuncts, column_id, filter, static_cast(rows - *rows_read), &loop_filter, &loop_rows, &eof, &loop_used)); if (!loop_used) { // A fallback is safe only before this call has consumed a logical row. Plain-only @@ -584,10 +584,11 @@ Status NativeColumnReader::select_with_dictionary_filter(const SelectionVector& return Status::OK(); } -Status NativeColumnReader::select_with_plain_filter( - const SelectionVector& selection, uint16_t selected_rows, int64_t batch_rows, - const std::vector& predicates, IColumn::Filter* row_filter, - bool* used_filter) { +Status NativeColumnReader::select_with_plain_filter(const SelectionVector& selection, + uint16_t selected_rows, int64_t batch_rows, + const VExprSPtrs& conjuncts, int column_id, + IColumn::Filter* row_filter, + bool* used_filter) { DORIS_CHECK(row_filter != nullptr); DORIS_CHECK(used_filter != nullptr); RETURN_IF_ERROR(validate_selected_span(batch_rows)); @@ -595,8 +596,8 @@ Status NativeColumnReader::select_with_plain_filter( const uint8_t* filter_data = nullptr; RETURN_IF_ERROR(selection.materialize_filter(selected_rows, batch_rows, &filter_data)); int64_t rows_read = 0; - RETURN_IF_ERROR(read_with_plain_filter(batch_rows, filter_data, selected_rows == 0, predicates, - row_filter, &rows_read, used_filter)); + RETURN_IF_ERROR(read_with_plain_filter(batch_rows, filter_data, selected_rows == 0, conjuncts, + column_id, row_filter, &rows_read, used_filter)); if (!*used_filter) { return Status::OK(); } @@ -680,9 +681,6 @@ int64_t NativeColumnReader::sync_native_profile() { COUNTER_UPDATE(_profile.decode_null_map_time, stats.decode_null_map_time - reported.decode_null_map_time); } - if (_profile.convert_time != nullptr) { - COUNTER_UPDATE(_profile.convert_time, stats.convert_time - reported.convert_time); - } if (_profile.materialization_time != nullptr) { COUNTER_UPDATE(_profile.materialization_time, stats.materialization_time - reported.materialization_time); diff --git a/be/src/format_v2/parquet/reader/native_column_reader.h b/be/src/format_v2/parquet/reader/native_column_reader.h index 9716f1adb180b8..4e504d3b2d7492 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.h +++ b/be/src/format_v2/parquet/reader/native_column_reader.h @@ -89,8 +89,7 @@ class NativeColumnReader final : public ParquetColumnReader { MutableColumnPtr& column, IColumn::Filter* row_filter, bool* used_filter) override; Status select_with_plain_filter(const SelectionVector& selection, uint16_t selected_rows, - int64_t batch_rows, - const std::vector& predicates, + int64_t batch_rows, const VExprSPtrs& conjuncts, int column_id, IColumn::Filter* row_filter, bool* used_filter) override; void flush_profile() override; bool crossed_page_since_last_batch() override; @@ -113,7 +112,7 @@ class NativeColumnReader final : public ParquetColumnReader { MutableColumnPtr& column, const DataTypePtr& output_type, bool dictionary_ids, int64_t* rows_read); Status read_with_plain_filter(int64_t rows, const uint8_t* filter_data, bool filter_all, - const std::vector& predicates, + const VExprSPtrs& conjuncts, int column_id, IColumn::Filter* row_filter, int64_t* rows_read, bool* used_filter); int64_t sync_native_profile(); @@ -122,7 +121,7 @@ class NativeColumnReader final : public ParquetColumnReader { void advance_selected_span(int64_t rows); // Native ParquetColumnReader keeps a reference to RowRanges; declare it before the reader. - ::doris::RowRanges _row_ranges; + segment_v2::RowRanges _row_ranges; std::set _projected_column_ids; std::set _filter_column_ids; const std::unordered_map* _offset_indexes = nullptr; diff --git a/be/src/format_v2/parquet/reader/plain_fixed_predicate.h b/be/src/format_v2/parquet/reader/plain_fixed_predicate.h deleted file mode 100644 index 0596ff6718d5c8..00000000000000 --- a/be/src/format_v2/parquet/reader/plain_fixed_predicate.h +++ /dev/null @@ -1,125 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -#include "common/status.h" -#include "util/unaligned.h" - -namespace doris::format::parquet { - -enum class PlainFixedPredicateOp : uint8_t { EQ, NE, LT, LE, GT, GE }; -enum class PlainFixedPredicateType : uint8_t { INT32, INT64, FLOAT, DOUBLE }; - -// A compiled predicate over the physical little-endian values of a fixed-width PLAIN page. -// Compilation is deliberately restricted to expressions whose Doris value and Parquet physical -// value have identical comparison semantics; casts and logical-type conversions stay on the -// ordinary materialization path. -class PlainFixedPredicate { -public: - template - static PlainFixedPredicate create(PlainFixedPredicateType type, PlainFixedPredicateOp op, - T literal) { - PlainFixedPredicate predicate; - predicate._type = type; - predicate._op = op; - static_assert(sizeof(T) <= sizeof(predicate._literal)); - memcpy(predicate._literal.data(), &literal, sizeof(T)); - return predicate; - } - - size_t value_width() const { - switch (_type) { - case PlainFixedPredicateType::INT32: - case PlainFixedPredicateType::FLOAT: - return sizeof(uint32_t); - case PlainFixedPredicateType::INT64: - case PlainFixedPredicateType::DOUBLE: - return sizeof(uint64_t); - } - __builtin_unreachable(); - } - - PlainFixedPredicateType type() const { return _type; } - - // AND this predicate into matches. The caller owns NULL handling because Parquet omits NULLs - // from the physical value stream. - Status evaluate(const uint8_t* values, size_t num_values, size_t value_width, - uint8_t* matches) const { - if (UNLIKELY(value_width != this->value_width())) { - return Status::Corruption("PLAIN predicate width {} does not match expected {}", - value_width, this->value_width()); - } - switch (_type) { - case PlainFixedPredicateType::INT32: - return _evaluate(values, num_values, matches); - case PlainFixedPredicateType::INT64: - return _evaluate(values, num_values, matches); - case PlainFixedPredicateType::FLOAT: - return _evaluate(values, num_values, matches); - case PlainFixedPredicateType::DOUBLE: - return _evaluate(values, num_values, matches); - } - __builtin_unreachable(); - } - -private: - template - Status _evaluate(const uint8_t* values, size_t num_values, uint8_t* matches) const { - const T literal = unaligned_load(_literal.data()); - for (size_t row = 0; row < num_values; ++row) { - if (matches[row] == 0) { - continue; - } - const T value = unaligned_load(values + row * sizeof(T)); - bool keep = false; - switch (_op) { - case PlainFixedPredicateOp::EQ: - keep = value == literal; - break; - case PlainFixedPredicateOp::NE: - keep = value != literal; - break; - case PlainFixedPredicateOp::LT: - keep = value < literal; - break; - case PlainFixedPredicateOp::LE: - keep = value <= literal; - break; - case PlainFixedPredicateOp::GT: - keep = value > literal; - break; - case PlainFixedPredicateOp::GE: - keep = value >= literal; - break; - } - matches[row] = static_cast(keep); - } - return Status::OK(); - } - - PlainFixedPredicateType _type = PlainFixedPredicateType::INT32; - PlainFixedPredicateOp _op = PlainFixedPredicateOp::EQ; - std::array _literal {}; -}; - -} // namespace doris::format::parquet diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 715312ebaa2831..5f2a4d230909f5 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -36,13 +37,15 @@ #include "core/data_type/data_type_string.h" #include "core/data_type/data_type_struct.h" #include "core/data_type_serde/parquet_timestamp.h" +#include "exprs/vectorized_fn_call.h" +#include "exprs/vliteral.h" +#include "exprs/vslot_ref.h" #include "format_v2/parquet/reader/native/byte_array_dict_decoder.h" #include "format_v2/parquet/reader/native/column_reader.h" #include "format_v2/parquet/reader/native/decoder.h" #include "format_v2/parquet/reader/native/delta_bit_pack_decoder.h" #include "format_v2/parquet/reader/native/level_decoder.h" #include "format_v2/parquet/reader/native/page_reader.h" -#include "format_v2/parquet/reader/plain_fixed_predicate.h" #include "io/fs/buffered_reader.h" #include "util/block_compression.h" #include "util/coding.h" @@ -54,24 +57,117 @@ namespace doris::format::parquet::native { namespace { -TEST(ParquetV2NativeDecoderTest, PlainFixedPredicateEvaluatesPhysicalValuesWithoutColumn) { +VExprSPtr create_int32_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, int32_t literal, + bool literal_on_left = false) { + const auto int_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({int_type->to_thrift(), int_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + auto slot = VSlotRef::create_shared(column_id, column_id, -1, int_type, "plain_int"); + auto value = VLiteral::create_shared(int_type, Field::create_field(literal)); + if (literal_on_left) { + root->add_child(value); + root->add_child(slot); + } else { + root->add_child(slot); + root->add_child(value); + } + return root; +} + +VExprSPtr create_float_raw_comparison(int column_id, const std::string& function_name, + TExprOpcode::type opcode, float literal) { + const auto float_type = std::make_shared(); + TFunctionName fn_name; + fn_name.__set_function_name(function_name); + TFunction fn; + fn.__set_name(fn_name); + fn.__set_binary_type(TFunctionBinaryType::BUILTIN); + fn.__set_arg_types({float_type->to_thrift(), float_type->to_thrift()}); + fn.__set_ret_type(std::make_shared()->to_thrift()); + fn.__set_has_var_args(false); + TExprNode node; + node.__set_node_type(TExprNodeType::BINARY_PRED); + node.__set_opcode(opcode); + node.__set_type(std::make_shared()->to_thrift()); + node.__set_fn(fn); + node.__set_num_children(2); + node.__set_is_nullable(false); + auto root = VectorizedFnCall::create_shared(node); + root->add_child(VSlotRef::create_shared(column_id, column_id, -1, float_type, "plain_float")); + root->add_child(VLiteral::create_shared(float_type, Field::create_field(literal))); + return root; +} + +TEST(ParquetV2NativeDecoderTest, RawExprEvaluatesPhysicalValuesWithoutColumn) { const std::array values = {1, 4, 7, 10, 13, 16}; std::array matches {}; matches.fill(1); - const auto greater_equal = PlainFixedPredicate::create(PlainFixedPredicateType::INT32, - PlainFixedPredicateOp::GE, int32_t {7}); - const auto less_than = PlainFixedPredicate::create(PlainFixedPredicateType::INT32, - PlainFixedPredicateOp::LT, int32_t {16}); + const auto greater_equal = create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 7); + const auto less_than = create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 16); + const auto int_type = std::make_shared(); + ASSERT_TRUE(greater_equal->can_execute_on_raw_fixed_values(int_type, 0)); ASSERT_TRUE(greater_equal - .evaluate(reinterpret_cast(values.data()), values.size(), - sizeof(int32_t), matches.data()) + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) .ok()); ASSERT_TRUE(less_than - .evaluate(reinterpret_cast(values.data()), values.size(), - sizeof(int32_t), matches.data()) + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) .ok()); EXPECT_EQ(matches, (std::array {0, 0, 1, 1, 1, 0})); + + matches.fill(1); + const auto literal_less_than_slot = + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 7, true); + ASSERT_TRUE(literal_less_than_slot + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(int32_t), int_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1, 1, 1})); +} + +TEST(ParquetV2NativeDecoderTest, RawExprPreservesFloatNanOrdering) { + const float nan = std::numeric_limits::quiet_NaN(); + const std::array values {-1, 0, 1, nan}; + const auto float_type = std::make_shared(); + + std::array matches {}; + matches.fill(1); + const auto greater_than_zero = create_float_raw_comparison(0, "gt", TExprOpcode::GT, 0); + ASSERT_TRUE(greater_than_zero + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 1, 1})); + + matches.fill(1); + const auto equals_nan = create_float_raw_comparison(0, "eq", TExprOpcode::EQ, nan); + ASSERT_TRUE(equals_nan + ->execute_on_raw_fixed_values( + reinterpret_cast(values.data()), values.size(), + sizeof(float), float_type, 0, matches.data()) + .ok()); + EXPECT_EQ(matches, (std::array {0, 0, 0, 1})); } class RejectFixedConsumer final : public ParquetFixedValueConsumer { @@ -343,6 +439,7 @@ Status load_scripted_page(tparquet::PageHeader header, const std::vector(); field.repetition_level = 0; field.definition_level = 0; ParquetPageReadContext context(preload_page_cache, page_cache_file_key); @@ -666,7 +763,7 @@ Status materialize_selected_dictionary_strings(const std::vector& d return Status::OK(); } -TEST(ParquetV2NativeDecoderTest, PlainFixedPredicateMapsNullableSparseRowsDirectly) { +TEST(ParquetV2NativeDecoderTest, RawExprMapsNullableSparseRowsDirectly) { const std::vector physical_values {1, 4, 7, 10, 13}; constexpr size_t LOGICAL_VALUES = 7; tparquet::PageHeader header; @@ -691,6 +788,7 @@ TEST(ParquetV2NativeDecoderTest, PlainFixedPredicateMapsNullableSparseRowsDirect chunk.meta_data.__set_data_page_offset(0); NativeFieldSchema field; field.physical_type = tparquet::Type::INT32; + field.data_type = std::make_shared(); ParquetPageReadContext page_context(false, ""); ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, LOGICAL_VALUES, nullptr, page_context); @@ -703,17 +801,14 @@ TEST(ParquetV2NativeDecoderTest, PlainFixedPredicateMapsNullableSparseRowsDirect ASSERT_TRUE(filter.init(input_filter.data(), input_filter.size(), false).ok()); ColumnSelectVector select_vector; ASSERT_TRUE(select_vector.init(null_runs, LOGICAL_VALUES, nullptr, &filter, 0).ok()); - const std::vector predicates { - PlainFixedPredicate::create(PlainFixedPredicateType::INT32, PlainFixedPredicateOp::GE, - int32_t {4}), - PlainFixedPredicate::create(PlainFixedPredicateType::INT32, PlainFixedPredicateOp::LT, - int32_t {13})}; + const VExprSPtrs predicates {create_int32_raw_comparison(0, "ge", TExprOpcode::GE, 4), + create_int32_raw_comparison(0, "lt", TExprOpcode::LT, 13)}; NullMap selected_nulls; IColumn::Filter physical_matches; IColumn::Filter row_filter; bool used_filter = false; ASSERT_TRUE(chunk_reader - .filter_plain_values(predicates, select_vector, &selected_nulls, + .filter_plain_values(predicates, 0, select_vector, &selected_nulls, &physical_matches, &row_filter, &used_filter) .ok()); @@ -2951,8 +3046,8 @@ TEST(ParquetV2NativeDecoderTest, ComplexPageStatisticsPreservePerLeafCrossings) second_chunk.page_read_counter = 1; second_chunk.data_page_read_counter = 1; ColumnReader::ColumnStatistics combined; - ColumnReader::ColumnStatistics first(first_chunk, 0, 0); - ColumnReader::ColumnStatistics second(second_chunk, 0, 0); + ColumnReader::ColumnStatistics first(first_chunk, 0); + ColumnReader::ColumnStatistics second(second_chunk, 0); combined.merge(first); combined.merge(second); diff --git a/be/test/format_v2/parquet/parquet_reader_test.cpp b/be/test/format_v2/parquet/parquet_reader_test.cpp index ada2c17580a978..b63c161b0b6200 100644 --- a/be/test/format_v2/parquet/parquet_reader_test.cpp +++ b/be/test/format_v2/parquet/parquet_reader_test.cpp @@ -1584,7 +1584,7 @@ TEST_F(NewParquetReaderTest, NativeDecimalAndFixedBinaryMaterializeDirectly) { ASSERT_NE(profile.get_counter("LevelOnlyReadTime"), nullptr); EXPECT_EQ(profile.get_counter("LevelOnlyReadTime")->value(), 0); - ASSERT_NE(profile.get_counter("ConvertTime"), nullptr); + EXPECT_EQ(profile.get_counter("ConvertTime"), nullptr); ASSERT_NE(profile.get_counter("NativeReadCalls"), nullptr); EXPECT_GT(profile.get_counter("NativeReadCalls")->value(), 0); } diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 096dfaa24435b4..07b6cf0ed05de2 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -256,6 +256,52 @@ class Int32PairSumExpr final : public VExpr { const std::string _expr_name = "Int32PairSumExpr"; }; +class Int32DirectGreaterExpr final : public VExpr { +public: + Int32DirectGreaterExpr(int column_id, int32_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int32_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_INT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + DORIS_CHECK_EQ(value_width, sizeof(int32_t)); + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int32_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int32_t _lower_bound; + const std::string _expr_name = "Int32DirectGreaterExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); @@ -299,6 +345,11 @@ VExprContextSPtr create_int32_pair_sum_conjunct(int left_column_id, int right_co std::make_shared(left_column_id, right_column_id, upper_bound)); } +VExprContextSPtr create_int32_direct_greater_conjunct(int column_id, int32_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -1196,6 +1247,36 @@ TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); } +TEST_F(ParquetScanTest, PlainDirectPathKeepsPayloadForMultiColumnResidual) { + write_int_pair_parquet_file(_file_path, 6, false); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_direct_greater_conjunct(0, 2)); + request->conjuncts.push_back(create_int32_pair_sum_conjunct(0, 1, 42)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + ASSERT_TRUE(reader->get_block(&block, &rows, &eof).ok()); + ASSERT_EQ(rows, 1); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {30})); + EXPECT_EQ(block.get_by_position(0).column->size(), rows); + // A residual expression still consumes id, so raw filtering must leave its payload available. + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectBatches"), 0); +} + // Scenario: every physical batch in every row group is rejected. Predicate readers reach each row // group boundary, while the lazy score reader remains at row 0. The boundary reset must discard that // reader and its pending lag instead of issuing SkipRecords for values that can never be observed. diff --git a/docs/file-scanner-v2-code-review-guide.md b/docs/file-scanner-v2-code-review-guide.md index 134240c75e266f..82a3ba03667753 100644 --- a/docs/file-scanner-v2-code-review-guide.md +++ b/docs/file-scanner-v2-code-review-guide.md @@ -137,11 +137,10 @@ format-specific checklist when reviewing Parquet or ORC. Validate INT96 nanos-of-day before widened Julian-day arithmetic, reject unit scaling overflow, and enforce Doris year 0001-9999 before materialization. Conversion failures must follow the same strict/non-strict and dictionary-ID propagation rules as other direct types. -- Verify schema-change routing separately from physical decode. Integer, FLOAT-to-DOUBLE, decimal, - and string-family changes should use the direct target-SerDe path. Other supported logical casts - may use one persistent generic `ColumnTypeConverter` source column; its value/null-map sizes must - reset per batch while normal-size capacity is retained, oversized capacity must be released after - the top-level parent consumes the batch, and it must never become a decoder-facing ABI. +- Verify schema-change routing separately from physical decode. The reader must emit the projected + file type, while `ColumnMapper`/`TableReader` perform file-to-table casts after file predicates. + Any different requested type at the native reader boundary must fail as an invariant violation; + do not add a reader-local conversion column or decoder-facing conversion ABI. - Dictionary review must separate dictionary-entry IDs from logical rows and non-null payload ordinals. Materialize the typed dictionary once per generation through the same SerDe, validate every index before access, and invalidate cached dictionary state at Row Group/file/type changes. @@ -151,10 +150,12 @@ format-specific checklist when reviewing Parquet or ORC. DELTA_LENGTH_BYTE_ARRAY, DELTA_BYTE_ARRAY, and BYTE_STREAM_SPLIT. Filtering must advance encoded values without allocating output; null runs must append defaults without advancing payload. - For predicate-only fixed-width PLAIN primitives, allow direct comparison only after proving the - whole Column Chunk uses compatible PLAIN value pages and the expression has identical physical - comparison semantics. The fallback decision must precede definition-level consumption. Verify - sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed - comparisons, mixed-encoding fallback, and a stable row-shaped hidden-slot placeholder. + whole Column Chunk uses compatible PLAIN value pages and every Expr advertises raw fixed-value + evaluation with identical Doris comparison semantics, including NaN ordering. The fallback + decision must precede definition-level consumption. Disable the direct path when a residual or + delete conjunct still references the hidden slot, because it needs the materialized payload. + Verify sparse input selection, interleaved NULLs, reversed literal comparisons, multiple ANDed + comparisons, mixed-encoding fallback, residual slot reuse, and a stable row-shaped placeholder. - For a filtered scalar page fragment, require one SerDe entry and one batch-level selected-decode dispatch. Nullable selections must first map logical rows to selected non-NULL physical ranges, decode those ranges once, and restore NULL slots in place; falling back per NULL run is a review diff --git a/docs/file-scanner-v2-parquet-scan-design.md b/docs/file-scanner-v2-parquet-scan-design.md index e703c1d133bbb8..70c041a89ccd48 100644 --- a/docs/file-scanner-v2-parquet-scan-design.md +++ b/docs/file-scanner-v2-parquet-scan-design.md @@ -450,13 +450,11 @@ offsets, and coalesced survivor spans so string columns perform larger range cop `StringRef[]` staging array. Other binary encodings may use persistent references that remain valid only while the page or dictionary buffer is pinned by the persistent leaf reader. -The direct path covers identical logical types, string-family compatibility, decimal -precision/scale changes, integer changes, and FLOAT-to-DOUBLE widening. Less common table-schema -changes such as STRING-to-DATE or DECIMAL-to-STRING keep the generic `ColumnTypeConverter`: the -file logical SerDe first fills a reusable source Doris column, then the generic cast appends to the -requested column. -This compatibility path is deliberately separate from the decoder ABI and does not reintroduce a -physical-value batch or `PhysicalToLogicalConverter` into ordinary Parquet reads. +The direct path materializes the projected file type. Table-schema changes, including numeric +widening, decimal precision/scale changes, and string/date conversions, are represented by +`ColumnMapper` and executed by `TableReader` after file-local predicates finish. The native reader +rejects a different requested type because accepting it would move schema evolution into physical +decode and could change predicate semantics. No intermediate physical-value batch is introduced. `DecodedColumnView` is not the native Parquet decoder output ABI. It describes already decoded physical values and is useful to generic format conversion code, but routing every Parquet value From 5d2856a2741ed31d2950a47975066497508e0014 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 22:41:13 +0800 Subject: [PATCH 33/34] [fix](be) Harden Parquet V2 page decoding edge cases ### What problem does this PR solve? Issue Number: None Problem Summary: Parquet V2 could send UINT32 physical values through a width-incompatible raw PLAIN predicate path, reject valid all-NULL pages whose value section is omitted, and terminate the BE when untrusted footer encoding metadata omitted a later non-PLAIN page. Require an identity-preserving raw width, represent level-only nullable pages with an explicit zero-value decoder state, and return corruption after a direct PLAIN scan has already advanced. ### Release note None ### Check List (For Author) - Test: Unit Test - ParquetV2NativeDecoderTest.* and ParquetScanTest.* (112 tests) - Behavior changed: Yes; malformed or width-incompatible Parquet inputs now take the typed fallback or return a query error instead of failing valid reads or terminating the BE. - Does this need documentation: No --- .../reader/native/column_chunk_reader.cpp | 66 +++++- .../reader/native/column_chunk_reader.h | 2 + .../parquet/reader/native_column_reader.cpp | 11 +- .../format_v2/parquet/native_decoder_test.cpp | 117 +++++++++ .../format_v2/parquet/parquet_scan_test.cpp | 224 ++++++++++++++++++ 5 files changed, 410 insertions(+), 10 deletions(-) diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp index fb4fdde26a48d3..34e940df72f780 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.cpp @@ -163,6 +163,18 @@ bool validate_offset_index(const tparquet::OffsetIndex& index, const ColumnChunk namespace { +class EmptyValueSectionDecoder final : public Decoder { +public: + Status skip_values(size_t num_values) override { + if (UNLIKELY(num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } + return Status::OK(); + } +}; + Status append_v2_int96_datetime(ColumnDateTimeV2::Container& data, const ParquetInt96Timestamp& value, const cctz::time_zone& timezone) { @@ -1054,20 +1066,33 @@ Status ColumnChunkReader::load_page_data() { _current_encoding = encoding; // Reuse page decoder + Decoder* encoding_decoder = nullptr; if (_decoders.find(static_cast(encoding)) != _decoders.end()) { - _page_decoder = _decoders[static_cast(encoding)].get(); + encoding_decoder = _decoders[static_cast(encoding)].get(); } else { std::unique_ptr page_decoder; RETURN_IF_ERROR(Decoder::get_decoder(_metadata.type, encoding, page_decoder)); // Set type length page_decoder->set_type_length(_get_type_length()); _decoders[static_cast(encoding)] = std::move(page_decoder); - _page_decoder = _decoders[static_cast(encoding)].get(); + encoding_decoder = _decoders[static_cast(encoding)].get(); + } + _empty_value_section = _page_data.empty() && _max_def_level > 0; + if (_empty_value_section) { + // Nullable all-NULL pages legally contain only definition levels. Keep them decodable for + // every advertised encoding, but make a non-NULL definition level fail before stale decoder + // state from the preceding page can be consumed. + if (_empty_value_decoder == nullptr) { + _empty_value_decoder = std::make_unique(); + } + _page_decoder = _empty_value_decoder.get(); + } else { + _page_decoder = encoding_decoder; + // Encoding headers cannot legitimately advertise more physical values than the data page's + // logical value count; establish the bound before decoders inspect external counts. + _page_decoder->set_expected_values(_remaining_num_values); + RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); } - // Encoding headers cannot legitimately advertise more physical values than the data page's - // logical value count; establish the bound before decoders inspect external counts. - _page_decoder->set_expected_values(_remaining_num_values); - RETURN_IF_ERROR(_page_decoder->set_data(&_page_data)); _state = DATA_LOADED; return Status::OK(); @@ -1275,6 +1300,11 @@ Status ColumnChunkReader::skip_values(size_t num_va _remaining_num_values, num_values); } if (skip_data) { + if (UNLIKELY(_empty_value_section && num_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + num_values); + } SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); RETURN_IF_ERROR(_page_decoder->skip_values(num_values)); } @@ -1291,8 +1321,14 @@ Status ColumnChunkReader::materialize_values( return Status::OK(); } SCOPED_RAW_TIMER(&_chunk_statistics.decode_value_time); + const size_t physical_values = select_vector.num_values() - select_vector.num_nulls(); + if (UNLIKELY(_empty_value_section && physical_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + physical_values); + } if (UNLIKELY((doris_column->is_column_dictionary() || context.dictionary_index_only) && - !_has_dict)) { + !_has_dict && physical_values != 0)) { return Status::IOError("Not dictionary coded"); } if (UNLIKELY(_remaining_num_values < select_vector.num_values())) { @@ -1351,6 +1387,17 @@ bool ColumnChunkReader::can_filter_plain_values( _metadata.type != tparquet::Type::FLOAT && _metadata.type != tparquet::Type::DOUBLE)) { return false; } + const auto primitive_type = remove_nullable(_field_schema->data_type)->get_primitive_type(); + const bool has_identity_width = + (_metadata.type == tparquet::Type::INT32 && primitive_type == TYPE_INT) || + (_metadata.type == tparquet::Type::INT64 && primitive_type == TYPE_BIGINT) || + (_metadata.type == tparquet::Type::FLOAT && primitive_type == TYPE_FLOAT) || + (_metadata.type == tparquet::Type::DOUBLE && primitive_type == TYPE_DOUBLE); + if (!has_identity_width) { + // Raw predicates consume the physical Parquet width. Logical conversions such as UINT32 + // to BIGINT must stay on the typed path or a four-byte value is interpreted as eight bytes. + return false; + } return std::ranges::all_of(conjuncts, [&](const auto& conjunct) { return conjunct != nullptr && conjunct->can_execute_on_raw_fixed_values(_field_schema->data_type, column_id); @@ -1415,6 +1462,11 @@ Status ColumnChunkReader::filter_plain_values( DORIS_CHECK_EQ(selection.total_values, select_vector.num_values() - select_vector.num_nulls()); DORIS_CHECK_EQ(selected_nulls->size(), select_vector.num_values() - select_vector.num_filtered()); + if (UNLIKELY(_empty_value_section && selection.total_values != 0)) { + return Status::Corruption( + "Parquet definition levels require {} values from an empty value section", + selection.total_values); + } physical_matches->clear(); if (selection.selected_values == 0) { diff --git a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h index 807a2531d2cfe9..02634f96d8038d 100644 --- a/be/src/format_v2/parquet/reader/native/column_chunk_reader.h +++ b/be/src/format_v2/parquet/reader/native/column_chunk_reader.h @@ -349,6 +349,8 @@ class ColumnChunkReader { bool _has_dict = false; bool _nested_row_started = false; Decoder* _page_decoder = nullptr; + std::unique_ptr _empty_value_decoder; + bool _empty_value_section = false; tparquet::Encoding::type _current_encoding = tparquet::Encoding::PLAIN; // Map: encoding -> Decoder // Plain or Dictionary encoding. If the dictionary grows too big, the encoding will fall back to the plain encoding diff --git a/be/src/format_v2/parquet/reader/native_column_reader.cpp b/be/src/format_v2/parquet/reader/native_column_reader.cpp index 355dd79f5422e2..abed29d70253ce 100644 --- a/be/src/format_v2/parquet/reader/native_column_reader.cpp +++ b/be/src/format_v2/parquet/reader/native_column_reader.cpp @@ -365,9 +365,14 @@ Status NativeColumnReader::read_with_plain_filter(int64_t rows, const uint8_t* f conjuncts, column_id, filter, static_cast(rows - *rows_read), &loop_filter, &loop_rows, &eof, &loop_used)); if (!loop_used) { - // A fallback is safe only before this call has consumed a logical row. Plain-only - // chunk validation makes the decision stable for all later page fragments. - DORIS_CHECK_EQ(*rows_read, 0); + if (UNLIKELY(*rows_read != 0)) { + // Footer encoding lists are untrusted. Once a prior page advanced the cursor, a + // typed fallback would restart the request at the wrong row, so reject the file + // instead of terminating the BE or returning shifted results. + return Status::Corruption( + "Parquet PLAIN predicate encoding changed after {} rows for column {}", + *rows_read, _name); + } row_filter->clear(); return Status::OK(); } diff --git a/be/test/format_v2/parquet/native_decoder_test.cpp b/be/test/format_v2/parquet/native_decoder_test.cpp index 5f2a4d230909f5..bcf3e851d765c4 100644 --- a/be/test/format_v2/parquet/native_decoder_test.cpp +++ b/be/test/format_v2/parquet/native_decoder_test.cpp @@ -404,6 +404,89 @@ std::vector serialize_page(tparquet::PageHeader header, return bytes; } +Status materialize_level_only_page(bool data_page_v2, tparquet::Type::type physical_type, + tparquet::Encoding::type encoding, bool all_null) { + constexpr size_t VALUE_COUNT = 3; + const std::vector encoded_levels {static_cast(VALUE_COUNT << 1), + static_cast(all_null ? 0 : 1)}; + std::vector payload; + tparquet::PageHeader header; + if (data_page_v2) { + payload = encoded_levels; + header.type = tparquet::PageType::DATA_PAGE_V2; + header.__isset.data_page_header_v2 = true; + header.data_page_header_v2.__set_num_values(VALUE_COUNT); + header.data_page_header_v2.__set_num_nulls(all_null ? VALUE_COUNT : 0); + header.data_page_header_v2.__set_num_rows(VALUE_COUNT); + header.data_page_header_v2.__set_encoding(encoding); + header.data_page_header_v2.__set_definition_levels_byte_length(encoded_levels.size()); + header.data_page_header_v2.__set_repetition_levels_byte_length(0); + header.data_page_header_v2.__set_is_compressed(false); + } else { + payload.resize(sizeof(uint32_t)); + encode_fixed32_le(payload.data(), encoded_levels.size()); + payload.insert(payload.end(), encoded_levels.begin(), encoded_levels.end()); + header.type = tparquet::PageType::DATA_PAGE; + header.__isset.data_page_header = true; + header.data_page_header.__set_num_values(VALUE_COUNT); + header.data_page_header.__set_encoding(encoding); + header.data_page_header.__set_definition_level_encoding(tparquet::Encoding::RLE); + header.data_page_header.__set_repetition_level_encoding(tparquet::Encoding::RLE); + } + header.__set_compressed_page_size(payload.size()); + header.__set_uncompressed_page_size(payload.size()); + + auto bytes = serialize_page(header, payload); + MemoryBufferedReader reader(bytes); + tparquet::ColumnChunk chunk; + chunk.meta_data.__set_type(physical_type); + chunk.meta_data.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + chunk.meta_data.__set_num_values(VALUE_COUNT); + chunk.meta_data.__set_total_compressed_size(bytes.size()); + chunk.meta_data.__set_data_page_offset(0); + NativeFieldSchema field; + field.physical_type = physical_type; + if (physical_type == tparquet::Type::BOOLEAN) { + field.data_type = std::make_shared(); + } else { + field.data_type = std::make_shared(); + } + field.definition_level = 1; + field.parquet_schema.__set_type(physical_type); + field.parquet_schema.__set_repetition_type(tparquet::FieldRepetitionType::OPTIONAL); + ParquetPageReadContext page_context(false, ""); + ColumnChunkReader chunk_reader(&reader, &chunk, &field, nullptr, VALUE_COUNT, + nullptr, page_context); + RETURN_IF_ERROR(chunk_reader.init()); + const auto load_status = chunk_reader.load_page_data(); + EXPECT_TRUE(load_status.ok()) << load_status; + RETURN_IF_ERROR(load_status); + + level_t level = -1; + EXPECT_EQ(chunk_reader.def_level_decoder().get_next_run(&level, VALUE_COUNT), VALUE_COUNT); + EXPECT_EQ(level, all_null ? 0 : 1); + FilterMap filter; + RETURN_IF_ERROR(filter.init(nullptr, VALUE_COUNT, false)); + NullMap null_map; + ColumnSelectVector select_vector; + const std::vector null_runs {static_cast(all_null ? 0 : VALUE_COUNT), + static_cast(all_null ? VALUE_COUNT : 0)}; + RETURN_IF_ERROR(select_vector.init(null_runs, VALUE_COUNT, &null_map, &filter, 0)); + auto column = field.data_type->create_column(); + ParquetDecodeContext decode_context; + static const auto utc = cctz::utc_time_zone(); + RETURN_IF_ERROR(init_decode_context_for_test(field, &utc, &decode_context)); + ParquetMaterializationState state; + state.enable_strict_mode = true; + const auto status = chunk_reader.materialize_values(column, *field.data_type->get_serde(), + decode_context, state, select_vector); + if (status.ok()) { + EXPECT_EQ(column->size(), VALUE_COUNT); + EXPECT_EQ(null_map, NullMap(VALUE_COUNT, all_null ? 1 : 0)); + } + return status; +} + Status load_scripted_page(tparquet::PageHeader header, const std::vector& payload, tparquet::CompressionCodec::type codec, bool preload_page_cache = false) { std::vector bytes; @@ -2588,6 +2671,40 @@ TEST(ParquetV2NativeDecoderTest, FlatPagesRejectLogicalAndPhysicalCardinalityMis EXPECT_TRUE(status.is()) << status; } +TEST(ParquetV2NativeDecoderTest, LevelOnlyAllNullPagesInitializeZeroValueDecoders) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, true); + EXPECT_TRUE(status.ok()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + +TEST(ParquetV2NativeDecoderTest, LevelOnlyPagesRejectDefinitionLevelsThatRequireValues) { + const std::array, 3> encodings {{ + {tparquet::Type::INT32, tparquet::Encoding::RLE_DICTIONARY}, + {tparquet::Type::INT32, tparquet::Encoding::DELTA_BINARY_PACKED}, + {tparquet::Type::BOOLEAN, tparquet::Encoding::RLE}, + }}; + for (const bool data_page_v2 : {false, true}) { + for (const auto& [physical_type, encoding] : encodings) { + const auto status = + materialize_level_only_page(data_page_v2, physical_type, encoding, false); + EXPECT_TRUE(status.is()) + << "v2=" << data_page_v2 << ", encoding=" << tparquet::to_string(encoding) + << ": " << status; + } + } +} + TEST(ParquetV2NativeDecoderTest, NestedV2PageRejectsMissingAdvertisedRowStarts) { tparquet::PageHeader header; header.type = tparquet::PageType::DATA_PAGE_V2; diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 07b6cf0ed05de2..82f322d36458ea 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -23,9 +23,11 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -61,6 +63,8 @@ #include "storage/index/zone_map/zonemap_eval_context.h" #include "storage/index/zone_map/zonemap_filter_result.h" #include "storage/utils.h" +#include "util/coding.h" +#include "util/thrift_util.h" namespace doris { namespace { @@ -76,6 +80,13 @@ const ColumnInt32& int32_data_column(const IColumn& column) { return assert_cast(column); } +const ColumnInt64& int64_data_column(const IColumn& column) { + if (const auto* nullable_column = check_and_get_column(&column)) { + return assert_cast(nullable_column->get_nested_column()); + } + return assert_cast(column); +} + const ColumnString& string_data_column(const IColumn& column) { if (const auto* nullable_column = check_and_get_column(&column)) { return assert_cast(nullable_column->get_nested_column()); @@ -302,6 +313,54 @@ class Int32DirectGreaterExpr final : public VExpr { const std::string _expr_name = "Int32DirectGreaterExpr"; }; +class Int64DirectGreaterExpr final : public VExpr { +public: + Int64DirectGreaterExpr(int column_id, int64_t lower_bound) + : VExpr(std::make_shared(), false), + _column_id(column_id), + _lower_bound(lower_bound) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block* block, const Selector*, size_t count, + ColumnPtr& result_column) const override { + DORIS_CHECK(block != nullptr); + const auto& input = int64_data_column(*block->get_by_position(_column_id).column); + auto result = ColumnUInt8::create(count, 0); + for (size_t row = 0; row < count; ++row) { + result->get_data()[row] = input.get_element(row) > _lower_bound; + } + result_column = std::move(result); + return Status::OK(); + } + + bool can_execute_on_raw_fixed_values(const DataTypePtr& data_type, + int column_id) const override { + return column_id == _column_id && + remove_nullable(data_type)->get_primitive_type() == TYPE_BIGINT; + } + + Status execute_on_raw_fixed_values(const uint8_t* values, size_t num_values, size_t value_width, + const DataTypePtr&, int, uint8_t* matches) const override { + if (value_width != sizeof(int64_t)) { + return Status::Corruption("BIGINT raw predicate received {}-byte values", value_width); + } + for (size_t row = 0; row < num_values; ++row) { + matches[row] &= unaligned_load(values + row * sizeof(int64_t)) > _lower_bound; + } + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + int64_t _lower_bound; + const std::string _expr_name = "Int64DirectGreaterExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); @@ -350,6 +409,11 @@ VExprContextSPtr create_int32_direct_greater_conjunct(int column_id, int32_t low std::make_shared(column_id, lower_bound)); } +VExprContextSPtr create_int64_direct_greater_conjunct(int column_id, int64_t lower_bound) { + return VExprContext::create_shared( + std::make_shared(column_id, lower_bound)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -370,6 +434,14 @@ std::shared_ptr build_int32_array(const std::vector& valu return finish_array(&builder); } +std::shared_ptr build_uint32_array(const std::vector& values) { + arrow::UInt32Builder builder; + for (const auto value : values) { + EXPECT_TRUE(builder.Append(value).ok()); + } + return finish_array(&builder); +} + std::shared_ptr build_string_array(const std::vector& values) { arrow::StringBuilder builder; for (const auto& value : values) { @@ -488,6 +560,103 @@ void write_int_pair_parquet_file(const std::string& file_path, int64_t row_group write_table(file_path, table, row_group_size, false, false, enable_statistics); } +void write_uint32_pair_parquet_file(const std::string& file_path) { + auto schema = arrow::schema({arrow::field("id", arrow::uint32(), false), + arrow::field("score", arrow::int32(), false)}); + auto table = arrow::Table::Make(schema, {build_uint32_array({1, 2147483648U, 4294957294U}), + build_int32_array({10, 20, 30})}); + write_table(file_path, table, 3, false, false, false); +} + +std::vector serialize_test_page(tparquet::PageHeader header, + const std::vector& payload) { + std::vector bytes; + ThriftSerializer serializer(/*compact=*/true, 128); + DORIS_CHECK(serializer.serialize(&header, &bytes).ok()); + bytes.insert(bytes.end(), payload.begin(), payload.end()); + return bytes; +} + +void write_misdeclared_two_page_parquet_file(const std::string& file_path) { + const std::array first_values {1, 2}; + std::vector first_payload(sizeof(first_values)); + memcpy(first_payload.data(), first_values.data(), first_payload.size()); + tparquet::PageHeader first_header; + first_header.type = tparquet::PageType::DATA_PAGE_V2; + first_header.__set_compressed_page_size(first_payload.size()); + first_header.__set_uncompressed_page_size(first_payload.size()); + first_header.__isset.data_page_header_v2 = true; + first_header.data_page_header_v2.__set_num_values(first_values.size()); + first_header.data_page_header_v2.__set_num_nulls(0); + first_header.data_page_header_v2.__set_num_rows(first_values.size()); + first_header.data_page_header_v2.__set_encoding(tparquet::Encoding::PLAIN); + first_header.data_page_header_v2.__set_definition_levels_byte_length(0); + first_header.data_page_header_v2.__set_repetition_levels_byte_length(0); + first_header.data_page_header_v2.__set_is_compressed(false); + auto column_bytes = serialize_test_page(first_header, first_payload); + + auto node = ::parquet::schema::PrimitiveNode::Make("id", ::parquet::Repetition::REQUIRED, + ::parquet::Type::INT32); + ::parquet::ColumnDescriptor descriptor(node, 0, 0); + auto encoder = ::parquet::MakeTypedEncoder<::parquet::Int32Type>( + ::parquet::Encoding::DELTA_BINARY_PACKED, false, &descriptor); + const int32_t second_values[] = {3, 4}; + encoder->Put(second_values, std::size(second_values)); + auto second_buffer = encoder->FlushValues(); + std::vector second_payload(second_buffer->data(), + second_buffer->data() + second_buffer->size()); + tparquet::PageHeader second_header = first_header; + second_header.__set_compressed_page_size(second_payload.size()); + second_header.__set_uncompressed_page_size(second_payload.size()); + second_header.data_page_header_v2.__set_encoding(tparquet::Encoding::DELTA_BINARY_PACKED); + auto second_page = serialize_test_page(second_header, second_payload); + column_bytes.insert(column_bytes.end(), second_page.begin(), second_page.end()); + + tparquet::SchemaElement root; + root.__set_name("schema"); + root.__set_num_children(1); + tparquet::SchemaElement leaf; + leaf.__set_name("id"); + leaf.__set_type(tparquet::Type::INT32); + leaf.__set_repetition_type(tparquet::FieldRepetitionType::REQUIRED); + tparquet::ColumnMetaData column; + column.__set_type(tparquet::Type::INT32); + // Deliberately omit DELTA_BINARY_PACKED to emulate untrusted/incomplete footer metadata. + column.__set_encodings({tparquet::Encoding::PLAIN, tparquet::Encoding::RLE}); + column.__set_path_in_schema({"id"}); + column.__set_codec(tparquet::CompressionCodec::UNCOMPRESSED); + column.__set_num_values(4); + column.__set_total_uncompressed_size(column_bytes.size()); + column.__set_total_compressed_size(column_bytes.size()); + column.__set_data_page_offset(4); + tparquet::ColumnChunk chunk; + chunk.__set_file_offset(4); + chunk.__set_meta_data(column); + tparquet::RowGroup row_group; + row_group.__set_columns({chunk}); + row_group.__set_total_byte_size(column_bytes.size()); + row_group.__set_num_rows(4); + tparquet::FileMetaData metadata; + metadata.__set_version(2); + metadata.__set_schema({root, leaf}); + metadata.__set_num_rows(4); + metadata.__set_row_groups({row_group}); + + std::vector footer; + ThriftSerializer serializer(/*compact=*/true, 1024); + DORIS_CHECK(serializer.serialize(&metadata, &footer).ok()); + std::ofstream output(file_path, std::ios::binary | std::ios::trunc); + output.write("PAR1", 4); + output.write(reinterpret_cast(column_bytes.data()), column_bytes.size()); + output.write(reinterpret_cast(footer.data()), footer.size()); + std::array footer_size {}; + encode_fixed32_le(footer_size.data(), cast_set(footer.size())); + output.write(reinterpret_cast(footer_size.data()), footer_size.size()); + output.write("PAR1", 4); + output.close(); + DORIS_CHECK(output.good()); +} + void write_long_prefix_parquet_file(const std::string& file_path, size_t rows) { std::vector ids(rows); std::vector first(rows); @@ -1247,6 +1416,61 @@ TEST_F(ParquetScanTest, PredicateOnlyPlainComparisonUsesPhysicalDirectPath) { EXPECT_EQ(counter_value(profile, "PredicateCompactionBytes"), 0); } +TEST_F(ParquetScanTest, PredicateOnlyUint32FallsBackBeforeRawPlainDecode) { + write_uint32_pair_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + ASSERT_EQ(schema.size(), 2); + EXPECT_EQ(remove_nullable(schema[0].type)->get_primitive_type(), TYPE_BIGINT); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back( + create_int64_direct_greater_conjunct(0, std::numeric_limits::max())); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(rows, 2); + EXPECT_EQ(int32_data_column(*block.get_by_position(1).column).get_data(), + (ColumnInt32::Container {20, 30})); + EXPECT_EQ(counter_value(profile, "PlainPredicateDirectBatches"), 0); +} + +TEST_F(ParquetScanTest, PlainPredicateReportsFooterEncodingMismatchAfterProgress) { + write_misdeclared_two_page_parquet_file(_file_path); + RuntimeProfile profile("profile"); + auto reader = create_reader(0, -1, &profile); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + ASSERT_TRUE(request_builder.add_predicate_column(format::LocalColumnId(0)).ok()); + request->predicate_only_columns.push_back(format::LocalColumnId(0)); + request->conjuncts.push_back(create_int32_function_conjunct(0, "gt", TExprOpcode::GT, 0)); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + EXPECT_TRUE(status.is()) << status; + EXPECT_NE(status.to_string().find("encoding"), std::string::npos) << status; +} + TEST_F(ParquetScanTest, PlainDirectPathKeepsPayloadForMultiColumnResidual) { write_int_pair_parquet_file(_file_path, 6, false); RuntimeProfile profile("profile"); From bdb4f3d4dcd28376a150ec63f3cd9a0bba68f055 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 20 Jul 2026 23:24:19 +0800 Subject: [PATCH 34/34] [fix](be) Preserve signed Parquet V2 virtual column ids --- be/src/format_v2/parquet/parquet_scan.cpp | 44 +++++++------- be/src/format_v2/parquet/parquet_scan.h | 10 ++-- .../format_v2/parquet/parquet_scan_test.cpp | 57 +++++++++++++++++++ 3 files changed, 86 insertions(+), 25 deletions(-) diff --git a/be/src/format_v2/parquet/parquet_scan.cpp b/be/src/format_v2/parquet/parquet_scan.cpp index 981541c37732c3..4a12153cd30845 100644 --- a/be/src/format_v2/parquet/parquet_scan.cpp +++ b/be/src/format_v2/parquet/parquet_scan.cpp @@ -1061,16 +1061,16 @@ Status ParquetScanScheduler::open_next_row_group( _merge_read_slice_size); for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); + const auto local_id = col.column_id(); if (_current_predicate_columns.contains(local_id)) { continue; } - if (local_id == format::ROW_POSITION_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { _current_predicate_columns[local_id] = std::make_unique( _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); _current_predicate_columns[local_id] = std::make_unique( *_global_rowid_context, _current_row_group_first_row, @@ -1078,8 +1078,9 @@ Status ParquetScanScheduler::open_next_row_group( continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; RETURN_IF_ERROR(NativeColumnReader::create( @@ -1100,24 +1101,25 @@ Status ParquetScanScheduler::open_next_row_group( file_context, file_schema, prefetch_columns, &_current_predicate_prefetched)); } for (const auto& col : request.non_predicate_columns) { - const auto local_id = col.local_id(); + const auto local_id = col.column_id(); if (request.is_count_star_placeholder(col.column_id())) { continue; } - if (local_id == format::ROW_POSITION_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::ROW_POSITION_COLUMN_ID)) { _current_non_predicate_columns[local_id] = std::make_unique( _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - if (local_id == format::GLOBAL_ROWID_COLUMN_ID) { + if (local_id == format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID)) { DORIS_CHECK(_global_rowid_context.has_value()); _current_non_predicate_columns[local_id] = std::make_unique( *_global_rowid_context, _current_row_group_first_row, _scan_profile.column_reader_profile); continue; } - DORIS_CHECK(local_id >= 0 && local_id < static_cast(file_schema.size())); - const auto& column_schema = file_schema[local_id]; + DORIS_CHECK(local_id.is_valid() && + local_id.value() < static_cast(file_schema.size())); + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); std::unique_ptr column_reader; RETURN_IF_ERROR(NativeColumnReader::create( @@ -1331,8 +1333,8 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( SCOPED_TIMER(_scan_profile.dict_filter_rewrite_time); for (const auto& col : request.predicate_columns) { - const auto local_id = col.local_id(); - if (local_id < 0 || local_id >= static_cast(file_schema.size())) { + const auto local_id = col.column_id(); + if (!local_id.is_valid() || local_id.value() >= static_cast(file_schema.size())) { continue; } const auto position_it = request.local_positions.find(col.column_id()); @@ -1348,7 +1350,7 @@ Status ParquetScanScheduler::prepare_current_dictionary_filters( // This optimization is deliberately limited to single-column predicates with a dictionary // evaluable part. Mixed AND predicates are split so dictionary-covered children run as a // dict-id prefilter and residual children keep the normal row-level expression path. - const auto& column_schema = file_schema[local_id]; + const auto& column_schema = file_schema[local_id.value()]; DORIS_CHECK(column_schema != nullptr); if (column_schema->leaf_column_id < 0 || column_schema->leaf_column_id >= static_cast(row_group_metadata.columns.size())) { @@ -1525,9 +1527,9 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, }; auto read_predicate_column = - [&](ParquetColumnReader* column_reader, size_t block_position, ColumnId local_id, - const VExprContextSPtrs* single_column_conjuncts, bool* used_dictionary_filter, - bool* used_plain_filter) -> Status { + [&](ParquetColumnReader* column_reader, size_t block_position, + format::LocalColumnId local_id, const VExprContextSPtrs* single_column_conjuncts, + bool* used_dictionary_filter, bool* used_plain_filter) -> Status { DORIS_CHECK(used_dictionary_filter != nullptr); DORIS_CHECK(used_plain_filter != nullptr); *used_dictionary_filter = false; @@ -1574,7 +1576,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, if (single_column_conjuncts != nullptr && !residual_predicate_positions.contains(block_position) && - request.is_predicate_only(format::LocalColumnId(cast_set(local_id)))) { + request.is_predicate_only(local_id)) { VExprSPtrs direct_conjuncts; direct_conjuncts.reserve(single_column_conjuncts->size()); std::ranges::transform(*single_column_conjuncts, std::back_inserter(direct_conjuncts), @@ -1734,7 +1736,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, auto read_all_predicate_columns = [&]() -> Status { for (const auto& [fid, column_reader] : _current_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); bool used_dictionary_filter = false; bool used_plain_filter = false; @@ -1767,7 +1769,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, const size_t position = ordered_positions[order_idx]; const size_t idx = _predicate_indices_by_position_scratch.at(position); const auto& col = request.predicate_columns[idx]; - const auto fid = col.local_id(); + const auto fid = col.column_id(); auto reader_it = _current_predicate_columns.find(fid); DORIS_CHECK(reader_it != _current_predicate_columns.end()); auto position_it = request.local_positions.find(col.column_id()); @@ -1822,7 +1824,7 @@ Status ParquetScanScheduler::read_filter_columns(int64_t batch_rows, remaining_order_idx < ordered_positions.size(); ++remaining_order_idx) { const size_t remaining_idx = _predicate_indices_by_position_scratch.at( ordered_positions[remaining_order_idx]); - const auto remaining_fid = request.predicate_columns[remaining_idx].local_id(); + const auto remaining_fid = request.predicate_columns[remaining_idx].column_id(); auto remaining_reader_it = _current_predicate_columns.find(remaining_fid); DORIS_CHECK(remaining_reader_it != _current_predicate_columns.end()); RETURN_IF_ERROR(remaining_reader_it->second->skip(batch_rows)); @@ -1995,7 +1997,7 @@ Status ParquetScanScheduler::read_current_row_group_batch( // selection vector. This also merges pending range gaps with fully filtered batches. RETURN_IF_ERROR(flush_pending_non_predicate_skip_rows()); for (const auto& [fid, column_reader] : _current_non_predicate_columns) { - auto position_it = request.local_positions.find(format::LocalColumnId(fid)); + auto position_it = request.local_positions.find(fid); DORIS_CHECK(position_it != request.local_positions.end()); const auto block_position = position_it->second.value(); auto column = file_block->get_by_position(block_position).column->assert_mutable(); diff --git a/be/src/format_v2/parquet/parquet_scan.h b/be/src/format_v2/parquet/parquet_scan.h index 2fcdf253b328bd..ff3930bf7e4ddd 100644 --- a/be/src/format_v2/parquet/parquet_scan.h +++ b/be/src/format_v2/parquet/parquet_scan.h @@ -246,13 +246,15 @@ class ParquetScanScheduler { bool _has_current_row_group = false; // Readers retain pointers into this immutable row-group map, so it must outlive both maps below. std::unordered_map _current_offset_indexes; - std::map> + // File-local ids are signed because virtual columns use reserved negative values. Keeping the + // typed id as the map key prevents GLOBAL_ROWID_COLUMN_ID from wrapping to a storage ColumnId. + std::map> _current_predicate_columns; // predicate ColumnReaders - std::map> + std::map> _current_non_predicate_columns; // non-predicate ColumnReaders - std::map + std::map _current_dictionary_filters; // local id -> dict entry bitmap - std::map>> + std::map>> _current_dictionary_residual_conjuncts; // local id -> row-level residual conjuncts int64_t _current_row_group_rows = 0; // current row group row count int _current_row_group_id = -1; // current row group id in parquet metadata diff --git a/be/test/format_v2/parquet/parquet_scan_test.cpp b/be/test/format_v2/parquet/parquet_scan_test.cpp index 82f322d36458ea..b44b4e6dcdae46 100644 --- a/be/test/format_v2/parquet/parquet_scan_test.cpp +++ b/be/test/format_v2/parquet/parquet_scan_test.cpp @@ -361,6 +361,28 @@ class Int64DirectGreaterExpr final : public VExpr { const std::string _expr_name = "Int64DirectGreaterExpr"; }; +class AlwaysTrueSingleColumnExpr final : public VExpr { +public: + explicit AlwaysTrueSingleColumnExpr(int column_id) + : VExpr(std::make_shared(), false), _column_id(column_id) {} + + const std::string& expr_name() const override { return _expr_name; } + + Status execute_column_impl(VExprContext*, const Block*, const Selector*, size_t count, + ColumnPtr& result_column) const override { + result_column = ColumnUInt8::create(count, 1); + return Status::OK(); + } + + void collect_slot_column_ids(std::set& column_ids) const override { + column_ids.insert(_column_id); + } + +private: + int _column_id; + const std::string _expr_name = "AlwaysTrueSingleColumnExpr"; +}; + VExprContextSPtr create_int32_zonemap_conjunct(int column_id, Int32ZoneMapExpr::Op op, int32_t value) { return VExprContext::create_shared(std::make_shared(column_id, op, value)); @@ -414,6 +436,10 @@ VExprContextSPtr create_int64_direct_greater_conjunct(int column_id, int64_t low std::make_shared(column_id, lower_bound)); } +VExprContextSPtr create_always_true_single_column_conjunct(int column_id) { + return VExprContext::create_shared(std::make_shared(column_id)); +} + int64_t counter_value(RuntimeProfile& profile, const std::string& name) { auto* counter = profile.get_counter(name); DORIS_CHECK(counter != nullptr); @@ -1227,6 +1253,37 @@ TEST_F(ParquetScanTest, GlobalRowIdUsesFileLocalPositionForScanRange) { EXPECT_EQ(row_ids, std::vector({2, 3})); } +TEST_F(ParquetScanTest, PredicateOnlyGlobalRowIdKeepsSignedFileLocalId) { + write_int_pair_parquet_file(_file_path, 6, false); + format::GlobalRowIdContext context {.version = 7, .backend_id = 123456789, .file_id = 42}; + auto reader = create_reader(0, -1, nullptr, context); + RuntimeState state {TQueryOptions(), TQueryGlobals()}; + ASSERT_TRUE(reader->init(&state).ok()); + + std::vector schema; + ASSERT_TRUE(reader->get_schema(&schema).ok()); + auto request = std::make_shared(); + format::FileScanRequestBuilder request_builder(request.get()); + const auto global_rowid = format::LocalColumnId(format::GLOBAL_ROWID_COLUMN_ID); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(0)).ok()); + ASSERT_TRUE(request_builder.add_non_predicate_column(format::LocalColumnId(1)).ok()); + ASSERT_TRUE(request_builder.add_predicate_column(global_rowid).ok()); + request->predicate_only_columns.push_back(global_rowid); + const auto global_rowid_position = request->local_positions.at(global_rowid); + request->conjuncts.push_back(create_always_true_single_column_conjunct( + cast_set(global_rowid_position.value()))); + ASSERT_TRUE(reader->open(request).ok()); + + Block block = build_file_block(schema); + size_t rows = 0; + bool eof = false; + const auto status = reader->get_block(&block, &rows, &eof); + ASSERT_TRUE(status.ok()) << status; + EXPECT_EQ(rows, 6); + EXPECT_EQ(int32_data_column(*block.get_by_position(0).column).get_data(), + (ColumnInt32::Container {1, 2, 3, 4, 5, 6})); +} + TEST_F(ParquetScanTest, EmptyScanPlanReturnsEofWithoutReadingColumns) { write_int_pair_parquet_file(_file_path, 2); auto reader = create_reader();