From 53e9b50f70a9367a29149d3c953e4f2d02100061 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Tue, 28 Jul 2026 11:01:20 +0800 Subject: [PATCH] feat(blob): support placeholder fallback for partial updates A data-evolution partial update rewrites only the touched rows of a blob column and records every untouched row as a placeholder entry (bin_length -2, no data bytes). The blob format writes such entries only for partial updates: a data-evolution blob-only column write enables the placeholder protocol (BlobDefs::kWritePlaceholderKey), under which a value equal to the placeholder sentinel is persisted as a bin_length -2 entry and a sentinel-prefixed real value is unescaped; any other write stores blob bytes verbatim, so user data can never be turned into a placeholder. The reader fails on placeholder entries unless the internal option BlobDefs::kEmitPlaceholderSentinelKey switches it to emit the sentinel for them, escaping stored values that start with the sentinel bytes so the fallback merge can never mistake real data for placeholders. BlobBunch keeps all max-sequence layers of a bunch, and the new BlobFallbackBatchReader resolves each row to the newest layer holding a real value: an explicitly written null wins over older layers, a row that is a placeholder in every layer degrades to a null blob while keeping its _ROW_ID and reporting -1 as its _SEQUENCE_NUMBER, resolved rows report their layer's sequence number, and row-range pushdown is honored including the row id ranges a layer does not cover. Co-Authored-By: Claude Fable 5 --- src/paimon/CMakeLists.txt | 2 + src/paimon/common/data/blob_defs.h | 56 ++ .../reader/blob_fallback_batch_reader.cpp | 440 ++++++++++++++++ .../reader/blob_fallback_batch_reader.h | 174 +++++++ .../blob_fallback_batch_reader_test.cpp | 362 +++++++++++++ src/paimon/core/append/append_only_writer.cpp | 13 +- src/paimon/core/append/append_only_writer.h | 2 +- .../core/io/blob_data_file_writer_factory.cpp | 14 +- .../core/io/blob_data_file_writer_factory.h | 6 +- .../complete_row_tracking_fields_reader.cpp | 15 +- src/paimon/core/mergetree/lookup_levels.cpp | 3 +- .../core/operation/abstract_split_read.cpp | 14 +- .../core/operation/abstract_split_read.h | 10 +- .../operation/data_evolution_split_read.cpp | 236 ++++++--- .../operation/data_evolution_split_read.h | 40 +- .../data_evolution_split_read_test.cpp | 141 +++-- .../core/operation/merge_file_split_read.cpp | 6 +- .../core/operation/raw_file_split_read.cpp | 3 +- .../format/blob/blob_file_batch_reader.cpp | 77 ++- .../format/blob/blob_file_batch_reader.h | 33 +- .../blob/blob_file_batch_reader_test.cpp | 47 +- src/paimon/format/blob/blob_format_writer.cpp | 27 +- src/paimon/format/blob/blob_format_writer.h | 12 +- .../format/blob/blob_format_writer_test.cpp | 365 +++++++++++-- src/paimon/format/blob/blob_reader_builder.h | 6 +- .../format/blob/blob_stats_extractor.cpp | 3 +- src/paimon/format/blob/blob_writer_builder.h | 6 +- test/inte/blob_table_inte_test.cpp | 481 ++++++++++++++++++ 28 files changed, 2366 insertions(+), 228 deletions(-) create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader.cpp create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader.h create mode 100644 src/paimon/common/reader/blob_fallback_batch_reader_test.cpp diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 409c58a50..09b813c36 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -130,6 +130,7 @@ set(PAIMON_COMMON_SRCS common/reader/predicate_batch_reader.cpp common/reader/prefetch_file_batch_reader_impl.cpp common/reader/reader_utils.cpp + common/reader/blob_fallback_batch_reader.cpp common/reader/blob_view_resolving_batch_reader.cpp common/reader/complete_row_kind_batch_reader.cpp common/reader/data_evolution_file_reader.cpp @@ -542,6 +543,7 @@ if(PAIMON_BUILD_TESTS) common/reader/prefetch_file_batch_reader_impl_test.cpp common/reader/reader_utils_test.cpp common/reader/complete_row_kind_batch_reader_test.cpp + common/reader/blob_fallback_batch_reader_test.cpp common/reader/blob_view_resolving_batch_reader_test.cpp common/reader/data_evolution_file_reader_test.cpp common/reader/data_evolution_array_test.cpp diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h index b8bb6f308..aac13afe2 100644 --- a/src/paimon/common/data/blob_defs.h +++ b/src/paimon/common/data/blob_defs.h @@ -17,6 +17,8 @@ #pragma once #include +#include +#include namespace paimon { @@ -45,6 +47,60 @@ class BlobDefs { /// A bin_length value of -1 in the index indicates a null blob entry. static constexpr int64_t kNullBinLength = -1; + /// A bin_length value of -2 in the index indicates a placeholder blob entry, written by + /// data-evolution partial updates for rows whose blob value is not updated. A placeholder + /// entry occupies no file space; readers must fall back to an older blob file covering the + /// same row to resolve the value. Aligned with Java's BlobFormatWriter.PLACE_HOLDER_LENGTH. + static constexpr int64_t kPlaceholderBinLength = -2; + /// Sentinel bytes standing for a placeholder blob value in two internal channels: + /// + /// - Write channel: a data-evolution partial update (a blob-only column write, see + /// kWritePlaceholderKey) marks a not-updated row with these bytes, and the blob format + /// writer persists it as a bin_length -2 entry. Use PlaceholderSentinelView() to build + /// such write arrays. Outside that mode the writer never interprets values, so arbitrary + /// user bytes can never be turned into a placeholder entry. + /// - Read channel: a placeholder-aware reader (see kEmitPlaceholderSentinelKey) emits these + /// bytes for -2 entries so the fallback merge can identify placeholders after the batch + /// has passed through schema-mapping readers. + /// + /// Both channels escape legitimate values so user bytes can never collide with the + /// sentinel: a real value whose bytes start with the sentinel travels with one extra copy + /// of the sentinel prepended, and the consuming end strips it (an exact match is a + /// placeholder, a longer match is unescaped). Sentinel bytes are never stored in blob files + /// and never returned to users. Layout: version(1) + magic "BLOBPLHD"(8). + static constexpr char kPlaceholderSentinel[] = {0x01, 'B', 'L', 'O', 'B', 'P', 'L', 'H', 'D'}; + static constexpr int32_t kPlaceholderSentinelLength = 9; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// reader emits kPlaceholderSentinel for placeholder entries (escaping real values as + /// described above) instead of failing on them. Only the data-evolution blob fallback read + /// path sets this. + static constexpr char kEmitPlaceholderSentinelKey[] = "blob.internal.emit-placeholder-sentinel"; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// format writer applies the placeholder protocol to incoming values (an exact + /// kPlaceholderSentinel match becomes a bin_length -2 entry, a longer match is unescaped). + /// Only set for data-evolution partial updates, i.e. blob-only column writes of a table + /// with data evolution enabled; all other writes store bytes verbatim. + static constexpr char kWritePlaceholderKey[] = "blob.internal.write-placeholder"; + + /// The sentinel bytes for building a data-evolution partial-update write array: a row equal + /// to this view is persisted as a placeholder entry (see kWritePlaceholderKey). + static std::string_view PlaceholderSentinelView() { + return {kPlaceholderSentinel, static_cast(kPlaceholderSentinelLength)}; + } + + static bool IsPlaceholderSentinel(const char* data, size_t size) { + return size == static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } + + /// True when the bytes start with the placeholder sentinel (an exact match included). A + /// longer match is an escaped real value: strip the leading sentinel to recover it. A + /// producer for either placeholder channel must prepend one sentinel copy to any real value + /// this predicate matches. + static bool HasPlaceholderSentinelPrefix(const char* data, size_t size) { + return size >= static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } /// Blob file format version. static constexpr int8_t kFileVersion = 1; /// Magic number identifying the start of each blob bin. diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp new file mode 100644 index 000000000..ae2547ffd --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp @@ -0,0 +1,440 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> BlobFallbackBatchReader::Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool) { + if (sequence_groups.size() < 2) { + return Status::Invalid( + "Blob fallback needs at least two sequence groups; a single group should be read " + "sequentially."); + } + if (read_schema == nullptr) { + return Status::Invalid("Blob fallback read schema cannot be nullptr."); + } + if (read_batch_size <= 0) { + return Status::Invalid(fmt::format( + "Blob fallback read batch size '{}' should be larger than zero", read_batch_size)); + } + int32_t blob_field_idx = -1; + for (int32_t i = 0; i < read_schema->num_fields(); i++) { + if (BlobUtils::IsBlobField(read_schema->field(i))) { + if (blob_field_idx != -1) { + return Status::Invalid( + "Blob fallback read schema should contain exactly one blob field."); + } + blob_field_idx = i; + } + } + if (blob_field_idx == -1) { + return Status::Invalid("Blob fallback read schema should contain a blob field."); + } + int32_t row_id_field_idx = read_schema->GetFieldIndex(SpecialFields::RowId().Name()); + int32_t seq_num_field_idx = read_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); + std::vector groups; + groups.reserve(sequence_groups.size()); + for (auto& segments : sequence_groups) { + if (segments.empty()) { + return Status::Invalid("Blob fallback sequence group should not be empty."); + } + for (const auto& segment : segments) { + if (segment.reader == nullptr && segment.gap_selected_ranges.empty()) { + return Status::Invalid( + "Blob fallback gap segment should cover at least one selected row id."); + } + } + GroupCursor cursor; + cursor.segments = std::move(segments); + groups.push_back(std::move(cursor)); + } + return std::unique_ptr( + new BlobFallbackBatchReader(std::move(groups), read_schema, blob_field_idx, + row_id_field_idx, seq_num_field_idx, read_batch_size, pool)); +} + +BlobFallbackBatchReader::BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool) + : groups_(std::move(groups)), + read_schema_(read_schema), + blob_field_idx_(blob_field_idx), + row_id_field_idx_(row_id_field_idx), + seq_num_field_idx_(seq_num_field_idx), + read_batch_size_(read_batch_size), + arrow_pool_(GetArrowPool(pool)) {} + +Result BlobFallbackBatchReader::FillWindow(size_t group_idx, int64_t want, + std::vector* chunks) { + GroupCursor& cursor = groups_[group_idx]; + int64_t collected = 0; + while (collected < want) { + if (!cursor.pending.empty()) { + const std::shared_ptr& front = cursor.pending.front(); + int64_t available = front->length() - cursor.pending_pos; + int64_t take = std::min(available, want - collected); + chunks->push_back(Chunk{front, cursor.pending_pos, take, {}, false}); + cursor.pending_pos += take; + collected += take; + if (cursor.pending_pos == front->length()) { + cursor.pending.pop_front(); + cursor.pending_pos = 0; + } + continue; + } + if (cursor.segment_idx >= cursor.segments.size()) { + // group exhausted; only the first group may define a shorter window + break; + } + Segment& segment = cursor.segments[cursor.segment_idx]; + if (segment.reader == nullptr) { + // gap segment: all rows are placeholders, stepped range by range so the row ids + // stay available for all-placeholder rows + if (cursor.gap_range_idx >= segment.gap_selected_ranges.size()) { + cursor.segment_idx++; + cursor.gap_range_idx = 0; + cursor.gap_range_pos = 0; + continue; + } + const Range& range = segment.gap_selected_ranges[cursor.gap_range_idx]; + int64_t remaining = range.Count() - cursor.gap_range_pos; + int64_t take = std::min(remaining, want - collected); + Chunk chunk{nullptr, 0, take, {}, false}; + if (row_id_field_idx_ >= 0) { + chunk.gap_row_ids.reserve(take); + for (int64_t k = 0; k < take; k++) { + chunk.gap_row_ids.push_back(range.from + cursor.gap_range_pos + k); + } + } + chunks->push_back(std::move(chunk)); + cursor.gap_range_pos += take; + collected += take; + if (cursor.gap_range_pos == range.Count()) { + cursor.gap_range_idx++; + cursor.gap_range_pos = 0; + } + continue; + } + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, + segment.reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + cursor.segment_idx++; + continue; + } + auto& [read_batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = read_batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector selected_array_vec, + ReaderUtils::GenerateFilteredArrayVector(src_array, bitmap)); + for (const auto& selected_array : selected_array_vec) { + if (selected_array->length() == 0) { + continue; + } + auto struct_array = std::dynamic_pointer_cast(selected_array); + if (struct_array == nullptr) { + return Status::Invalid("Blob fallback expects file readers to emit struct arrays."); + } + cursor.pending.push_back(std::move(struct_array)); + } + } + return collected; +} + +Result> BlobFallbackBatchReader::ComputePlaceholderFlags( + std::vector* chunks, int64_t row_count) const { + std::vector flags(row_count, false); + int64_t pos = 0; + for (auto& chunk : *chunks) { + if (chunk.array == nullptr) { + // gap rows stand for placeholders + std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true); + } else { + std::shared_ptr blob_col = chunk.array->field(blob_field_idx_); + auto binary_col = std::dynamic_pointer_cast(blob_col); + if (binary_col == nullptr) { + return Status::Invalid(fmt::format( + "Blob fallback expects the blob column to be large binary, but got {}", + blob_col->type()->ToString())); + } + for (int64_t k = 0; k < chunk.length; k++) { + int64_t idx = chunk.offset + k; + if (binary_col->IsNull(idx)) { + continue; + } + std::string_view value = binary_col->GetView(idx); + if (!BlobDefs::HasPlaceholderSentinelPrefix(value.data(), value.size())) { + continue; + } + if (BlobDefs::IsPlaceholderSentinel(value.data(), value.size())) { + flags[pos + k] = true; + } else { + // a real value escaped by the placeholder-aware blob reader + chunk.has_escaped = true; + } + } + } + pos += chunk.length; + } + return flags; +} + +Result> BlobFallbackBatchReader::UnescapeBlobSlice( + const std::shared_ptr& blob_col, int64_t offset, + int64_t length) const { + arrow::LargeBinaryBuilder builder(arrow_pool_.get()); + for (int64_t k = 0; k < length; k++) { + int64_t idx = offset + k; + if (blob_col->IsNull(idx)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + std::string_view value = blob_col->GetView(idx); + if (BlobDefs::HasPlaceholderSentinelPrefix(value.data(), value.size()) && + value.size() > static_cast(BlobDefs::kPlaceholderSentinelLength)) { + value.remove_prefix(BlobDefs::kPlaceholderSentinelLength); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(value.data(), value.size())); + } + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array)); + return array; +} + +Result> BlobFallbackBatchReader::AssembleRowIdRun( + const std::vector& chunks, int64_t run_start, int64_t run_end) const { + arrow::ArrayVector pieces; + int64_t pos = 0; + for (const auto& chunk : chunks) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array != nullptr) { + std::shared_ptr column = chunk.array->field(row_id_field_idx_); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } else { + arrow::Int64Builder builder(arrow_pool_.get()); + for (int64_t r = overlap_start; r < overlap_end; r++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(chunk.gap_row_ids[r - pos])); + } + std::shared_ptr piece; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&piece)); + pieces.push_back(std::move(piece)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + if (pieces.size() == 1) { + return pieces[0]; + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result> BlobFallbackBatchReader::AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const { + const auto row_count = static_cast(group_choice.size()); + arrow::ArrayVector pieces; + int64_t run_start = 0; + while (run_start < row_count) { + const int32_t group = group_choice[run_start]; + int64_t run_end = run_start + 1; + while (run_end < row_count && group_choice[run_end] == group) { + run_end++; + } + if (group < 0) { + // placeholder in every layer: the blob degrades to null; the row keeps its row id + // (taken from the newest group, which steps in lockstep), reports -1 as its + // sequence number, and returns null for every other field + if (field_idx == row_id_field_idx_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_id_piece, + AssembleRowIdRun(group_chunks[0], run_start, run_end)); + pieces.push_back(std::move(row_id_piece)); + } else if (field_idx == seq_num_field_idx_) { + arrow::Int64Scalar seq_scalar(-1); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr seq_piece, + arrow::MakeArrayFromScalar(seq_scalar, run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(seq_piece)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_piece, + arrow::MakeArrayOfNull(read_schema_->field(field_idx)->type(), + run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(null_piece)); + } + } else { + int64_t pos = 0; + for (const auto& chunk : group_chunks[group]) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array == nullptr) { + return Status::Invalid( + "Unexpected: a gap row was chosen as a blob fallback result."); + } + std::shared_ptr column = chunk.array->field(field_idx); + int64_t slice_offset = chunk.offset + (overlap_start - pos); + int64_t slice_length = overlap_end - overlap_start; + if (field_idx == blob_field_idx_ && chunk.has_escaped) { + auto binary_col = + std::dynamic_pointer_cast(column); + if (binary_col == nullptr) { + return Status::Invalid( + "Blob fallback expects the blob column to be large binary."); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr unescaped, + UnescapeBlobSlice(binary_col, slice_offset, slice_length)); + pieces.push_back(std::move(unescaped)); + } else { + pieces.push_back(column->Slice(slice_offset, slice_length)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + } + run_start = run_end; + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result BlobFallbackBatchReader::NextBatch() { + if (closed_) { + return Status::Invalid("blob fallback batch reader is closed"); + } + std::vector> group_chunks(groups_.size()); + // the first (newest) group defines the window; the others must step in lockstep + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, FillWindow(0, read_batch_size_, &group_chunks[0])); + for (size_t g = 1; g < groups_.size(); g++) { + int64_t expect = row_count == 0 ? 1 : row_count; + PAIMON_ASSIGN_OR_RAISE(int64_t got, FillWindow(g, expect, &group_chunks[g])); + int64_t expected = row_count == 0 ? 0 : row_count; + if (got != expected) { + return Status::Invalid(fmt::format( + "All sequence groups of a blob fallback read should have the same number of " + "rows: group {} yielded {} rows in a window of {}", + g, got, expected)); + } + } + if (row_count == 0) { + return BatchReader::MakeEofBatch(); + } + + std::vector> placeholder_flags(groups_.size()); + for (size_t g = 0; g < groups_.size(); g++) { + PAIMON_ASSIGN_OR_RAISE(placeholder_flags[g], + ComputePlaceholderFlags(&group_chunks[g], row_count)); + } + // per row, the first group in max-sequence order with a real entry wins; -1 means the row is + // a placeholder in every group + std::vector group_choice(row_count, -1); + for (int64_t r = 0; r < row_count; r++) { + for (size_t g = 0; g < groups_.size(); g++) { + if (!placeholder_flags[g][r]) { + group_choice[r] = static_cast(g); + break; + } + } + } + + arrow::ArrayVector columns; + columns.reserve(read_schema_->num_fields()); + for (int32_t field_idx = 0; field_idx < read_schema_->num_fields(); field_idx++) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr column, + AssembleColumn(field_idx, group_choice, group_chunks)); + columns.push_back(std::move(column)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(columns, read_schema_->fields())); + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*target_array, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); +} + +Result BlobFallbackBatchReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return BatchReader::MakeEofBatchWithBitmap(); + } + return ReaderUtils::AddAllValidBitmap(std::move(batch)); +} + +void BlobFallbackBatchReader::Close() { + for (auto& group : groups_) { + group.pending.clear(); + for (auto& segment : group.segments) { + if (segment.reader) { + segment.reader->Close(); + } + } + } + closed_ = true; +} + +std::shared_ptr BlobFallbackBatchReader::GetReaderMetrics() const { + auto metrics = std::make_shared(); + for (const auto& group : groups_) { + for (const auto& segment : group.segments) { + if (segment.reader) { + auto reader_metrics = segment.reader->GetReaderMetrics(); + if (reader_metrics) { + metrics->Merge(reader_metrics); + } + } + } + } + return metrics; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h new file mode 100644 index 000000000..040cf316c --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.h @@ -0,0 +1,174 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/utils/range.h" + +namespace arrow { +class Array; +class StructArray; +class LargeBinaryArray; +} // namespace arrow + +namespace paimon { + +/// Merges the blob files of one data-evolution blob bunch that span multiple max sequence +/// number layers, resolving placeholder entries row by row. Aligned with Java's +/// BlobFallbackRecordReader / AllPlaceholdersRecordReader. +/// +/// A data-evolution partial update rewrites only the touched rows of a blob column; the new blob +/// file records every untouched row as a placeholder entry. Reading therefore needs, per row, the +/// value from the newest layer that holds a real (non-placeholder) entry: +/// +/// 1. The caller groups the blob files by max sequence number (one group per layer, newest +/// first) and, inside each group, orders them by first row id. Row id ranges the group's +/// files do not cover are represented by gap segments, which stand for all-placeholder rows. +/// 2. All groups span the same overall row id range, so with the same row-ranges selection +/// applied they yield the same number of rows and can be stepped in lockstep. +/// 3. Each output row takes the first group, in max-sequence order, whose row is not a +/// placeholder. Placeholder rows are identified by the BlobDefs::kPlaceholderSentinel bytes, +/// emitted by the blob format reader when BlobDefs::kEmitPlaceholderSentinelKey is set; a +/// real value starting with those bytes arrives escaped by one extra sentinel copy, which +/// this reader strips (see BlobDefs::HasPlaceholderSentinelPrefix). +/// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its +/// _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field. +class BlobFallbackBatchReader : public BatchReader { + public: + /// One piece of a sequence group: either a reader over a single blob file (already wrapped + /// with the usual per-file mapping readers and row selection), or a virtual gap standing for + /// row ids the group's files do not cover. Segments must be ordered by ascending row id. + struct Segment { + /// File segment: emits the rows of one blob file. Null for a gap segment. + std::unique_ptr reader; + /// Gap segment only: the selected row ids the gap emits (all placeholders), as sorted + /// disjoint ranges. Must not be empty for a gap segment. + std::vector gap_selected_ranges; + }; + + /// `sequence_groups` must be ordered by descending max sequence number and contain at least + /// two groups (a single group needs no fallback). `read_schema` is the schema every file + /// reader emits; it must contain exactly one blob field and may additionally contain the + /// row-tracking fields _ROW_ID and _SEQUENCE_NUMBER (completed per file by + /// CompleteRowTrackingFieldsBatchReader), which stay correct for rows that are a + /// placeholder in every layer. + static Result> Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool); + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + void Close() override; + + std::shared_ptr GetReaderMetrics() const override; + + private: + /// A run of consecutive rows already fetched from one group: `array` rows + /// [offset, offset + length) for a file segment, or `length` placeholder rows for a gap + /// segment (array is null). + struct Chunk { + std::shared_ptr array; + int64_t offset = 0; + int64_t length = 0; + /// Gap chunk only: the row id of each of the `length` rows, filled when the read + /// schema contains _ROW_ID so all-placeholder rows can keep their row id. + std::vector gap_row_ids; + /// File chunk only: true when the blob column holds at least one escaped value + /// (BlobDefs::HasPlaceholderSentinelPrefix but longer than the sentinel), which must be + /// unescaped before it reaches the output. + bool has_escaped = false; + }; + + /// Read progress of one sequence group. Movable only: deleting copy makes vector growth + /// move the cursors instead of instantiating Segment's deleted copy constructor. + struct GroupCursor { + GroupCursor() = default; + GroupCursor(const GroupCursor&) = delete; + GroupCursor& operator=(const GroupCursor&) = delete; + GroupCursor(GroupCursor&&) = default; + GroupCursor& operator=(GroupCursor&&) = default; + + std::vector segments; + size_t segment_idx = 0; + /// Position inside the current gap segment: index into gap_selected_ranges and the + /// number of rows already emitted from that range. + size_t gap_range_idx = 0; + int64_t gap_range_pos = 0; + /// Rows fetched from the current file segment but not yet consumed. + std::deque> pending; + int64_t pending_pos = 0; + }; + + BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool); + + /// Collects up to `want` rows from the group into chunks. Only the first group may come up + /// short (which defines the window size); any later group ending early is a misalignment. + Result FillWindow(size_t group_idx, int64_t want, std::vector* chunks); + + /// Flags each of the `row_count` window rows of the given chunks as placeholder or not, + /// and marks chunks holding escaped blob values (see Chunk::has_escaped). + Result> ComputePlaceholderFlags(std::vector* chunks, + int64_t row_count) const; + + /// Assembles one output column by stitching, per run of rows choosing the same group, + /// slices of that group's chunks (unescaping escaped blob values). For rows choosing no + /// group (placeholder in every layer), _ROW_ID is kept, _SEQUENCE_NUMBER becomes -1, and + /// every other field becomes null. + Result> AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const; + + /// Unescapes the blob column rows [offset, offset + length) of a file chunk: values + /// starting with the sentinel bytes lose the leading sentinel copy. + Result> UnescapeBlobSlice( + const std::shared_ptr& blob_col, int64_t offset, + int64_t length) const; + + /// Assembles the _ROW_ID values of rows [run_start, run_end) from the given group's chunks; + /// gap chunks contribute their synthesized row ids. + Result> AssembleRowIdRun(const std::vector& chunks, + int64_t run_start, + int64_t run_end) const; + + std::vector groups_; + std::shared_ptr read_schema_; + const int32_t blob_field_idx_; + /// Index of _ROW_ID / _SEQUENCE_NUMBER in the read schema, -1 when not read. + const int32_t row_id_field_idx_; + const int32_t seq_num_field_idx_; + const int32_t read_batch_size_; + std::shared_ptr arrow_pool_; + bool closed_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp new file mode 100644 index 000000000..6dbe761aa --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp @@ -0,0 +1,362 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/range.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +/// "PH" stands for a placeholder row (the sentinel bytes emitted by the placeholder-aware blob +/// reader), std::nullopt for null. +using BlobRows = std::vector>; + +class BlobFallbackBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + read_schema_ = arrow::schema(struct_type_->fields()); + } + + static std::string Sentinel() { + return std::string(BlobDefs::PlaceholderSentinelView()); + } + + std::shared_ptr MakeBlobStruct(const BlobRows& rows) const { + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + } + + /// One segment of a group: File(rows) for a file segment, Gap(n) for a placeholder gap + /// of n selected rows (with synthetic row ids when the schema does not read them, or the + /// given ranges via GapRanges). + struct SegmentSpec { + std::vector gap_ranges; + std::optional file_rows; + static SegmentSpec Gap(int64_t rows) { + return SegmentSpec{{Range(0, rows - 1)}, std::nullopt}; + } + static SegmentSpec GapRanges(std::vector ranges) { + return SegmentSpec{std::move(ranges), std::nullopt}; + } + static SegmentSpec File(BlobRows rows) { + return SegmentSpec{{}, std::move(rows)}; + } + }; + + std::vector MakeGroup(const std::vector& specs, + int32_t file_batch_size) const { + std::vector segments; + for (const auto& spec : specs) { + if (spec.file_rows) { + auto reader = std::make_unique(MakeBlobStruct(*spec.file_rows), + struct_type_, file_batch_size); + segments.push_back(BlobFallbackBatchReader::Segment{std::move(reader), {}}); + } else { + segments.push_back(BlobFallbackBatchReader::Segment{nullptr, spec.gap_ranges}); + } + } + return segments; + } + + /// Runs the fallback over the groups with several batch sizes and compares to expected rows. + void CheckFallback(const std::vector>& group_specs, + const BlobRows& expected_rows) const { + auto expected_array = MakeBlobStruct(expected_rows); + for (auto batch_size : arrow::internal::Iota(1, 8)) { + for (auto file_batch_size : {1, 3, 1024}) { + std::vector> groups; + groups.reserve(group_specs.size()); + for (const auto& specs : group_specs) { + groups.push_back(MakeGroup(specs, file_batch_size)); + } + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + + protected: + std::shared_ptr pool_; + std::shared_ptr struct_type_; + std::shared_ptr read_schema_; +}; + +TEST_F(BlobFallbackBatchReaderTest, TestBasicFallback) { + // newer layer updates row 1 only; rows 0 and 2 fall back to the older layer + CheckFallback( + {{SegmentSpec::File({"PH", "u1", "PH"})}, {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "u1", "b2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestGapPadding) { + // the newer layer only covers rows 2-3; the gaps stand for placeholders + CheckFallback({{SegmentSpec::Gap(2), SegmentSpec::File({"u2", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "b1", "u2", "b3"}); + // trailing gap + CheckFallback({{SegmentSpec::File({"PH", "u1"}), SegmentSpec::Gap(2)}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "u1", "b2", "b3"}); + // middle gap between two files of one layer + CheckFallback({{SegmentSpec::File({"u0"}), SegmentSpec::Gap(2), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"u0", "b1", "b2", "u3"}); + // a gap segment covering multiple disjoint selected ranges + CheckFallback({{SegmentSpec::GapRanges({Range(0, 0), Range(2, 2)}), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b2", "b3"})}}, + {"b0", "b2", "u3"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestAllPlaceholdersBecomesNull) { + // a row that is a placeholder in every layer degrades to null + CheckFallback({{SegmentSpec::File({"PH", "PH"})}, {SegmentSpec::File({"b0", "PH"})}}, + {"b0", std::nullopt}); + CheckFallback({{SegmentSpec::Gap(2)}, {SegmentSpec::File({"PH", "PH"})}}, + {std::nullopt, std::nullopt}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestNullIsNotPlaceholder) { + // a real null in a newer layer wins: null means "updated to null", not "not updated" + CheckFallback({{SegmentSpec::File({std::nullopt, "u1"})}, {SegmentSpec::File({"b0", "b1"})}}, + {std::nullopt, "u1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestEscapedSentinelIsNotPlaceholder) { + // a real value whose bytes equal the sentinel arrives escaped (sentinel + value) from the + // placeholder-aware blob reader; the merge must unescape it, not treat it as a placeholder + std::string escaped_sentinel = Sentinel() + Sentinel(); + CheckFallback( + {{SegmentSpec::File({"PH", "u1"})}, {SegmentSpec::File({escaped_sentinel, "b1"})}}, + {Sentinel(), "u1"}); + // an escaped sentinel-prefixed value unescapes to the original bytes, also when it wins as + // the newest layer + std::string escaped_prefixed = Sentinel() + Sentinel() + "suffix"; + CheckFallback( + {{SegmentSpec::File({escaped_prefixed, "PH"})}, {SegmentSpec::File({"b0", "b1"})}}, + {Sentinel() + "suffix", "b1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestThreeLayers) { + CheckFallback({{SegmentSpec::File({"PH", "PH", "u2"})}, + {SegmentSpec::File({"PH", "m1", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "m1", "u2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestLayeredFilesAndGaps) { + // mirrors the compacted-sequence-groups shape: layers partially cover [0, 9] + CheckFallback( + {{SegmentSpec::Gap(6), SegmentSpec::File({"u66", "PH"}), SegmentSpec::Gap(1), + SegmentSpec::File({"u69"})}, + {SegmentSpec::File({"u40", "PH", "PH", "PH"}), SegmentSpec::Gap(4), + SegmentSpec::File({"u48", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"})}}, + {"u40", "b1", "b2", "b3", "b4", "b5", "u66", "b7", "u48", "u69"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) { + // Row-tracking projections: resolved rows keep their layer's row id and sequence number; + // an all-placeholder row keeps its row id (here provided by the newest group's gap + // segment), reports -1 as its sequence number, and degrades the blob to null. Covers the + // schema variants {blob, _ROW_ID, _SEQUENCE_NUMBER}, {blob, _ROW_ID} and + // {blob, _SEQUENCE_NUMBER}. + struct RowSpec { + std::optional blob; + int64_t row_id; + int64_t seq_num; + }; + for (bool with_row_id : {true, false}) { + for (bool with_seq_num : {true, false}) { + if (!with_row_id && !with_seq_num) { + continue; + } + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)}; + if (with_row_id) { + fields.push_back(SpecialFields::RowId().field_); + } + if (with_seq_num) { + fields.push_back(SpecialFields::SequenceNumber().field_); + } + auto struct_type = arrow::struct_(fields); + auto schema = arrow::schema(fields); + + auto make_rows = [&](const std::vector& rows) { + std::vector> field_builders = { + std::make_shared()}; + for (size_t i = 1; i < fields.size(); i++) { + field_builders.push_back(std::make_shared()); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row.blob) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row.blob == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row.blob->data(), row.blob->size()).ok()); + } + int32_t next_field = 1; + if (with_row_id) { + auto builder = static_cast( + struct_builder.field_builder(next_field++)); + EXPECT_TRUE(builder->Append(row.row_id).ok()); + } + if (with_seq_num) { + auto builder = static_cast( + struct_builder.field_builder(next_field)); + EXPECT_TRUE(builder->Append(row.seq_num).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + }; + + for (auto batch_size : arrow::internal::Iota(1, 5)) { + for (auto file_batch_size : {1, 1024}) { + // newest layer (seq 20) covers only row 2; rows 0-1 are a gap + std::vector newest; + newest.push_back(BlobFallbackBatchReader::Segment{nullptr, {Range(0, 1)}}); + newest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique(make_rows({{"u2", 2, 20}}), + struct_type, file_batch_size), + {}}); + // oldest layer (seq 10) covers rows 0-2, row 1 is a placeholder there too + std::vector oldest; + oldest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique( + make_rows({{"b0", 0, 10}, {"PH", 1, 10}, {"PH", 2, 10}}), struct_type, + file_batch_size), + {}}); + std::vector> groups; + groups.push_back(std::move(newest)); + groups.push_back(std::move(oldest)); + + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFallbackBatchReader::Create(std::move(groups), schema, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + + // row 0 falls back to seq 10, row 1 is all-placeholder (null blob, row id + // kept, seq -1), row 2 takes seq 20 + auto expected_array = + make_rows({{"b0", 0, 10}, {std::nullopt, 1, -1}, {"u2", 2, 20}}); + auto expected_chunk_array = + std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "with_row_id=" << with_row_id << " with_seq_num=" << with_seq_num + << " batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + } +} + +TEST_F(BlobFallbackBatchReaderTest, TestMisalignedGroupsFail) { + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"PH", "u1", "PH"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b0", "b1"})}, 1024)); + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, 1024, pool_)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "same number of rows"); +} + +TEST_F(BlobFallbackBatchReaderTest, TestCreateValidation) { + // a single group needs no fallback + std::vector> single_group; + single_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(single_group), read_schema_, 1024, pool_), + "at least two sequence groups"); + + // the read schema must contain a blob field + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b1"})}, 1024)); + auto plain_schema = + arrow::schema({arrow::field("not_blob", arrow::large_binary(), /*nullable=*/true)}); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(groups), plain_schema, 1024, pool_), + "should contain a blob field"); + + // groups must not be empty + std::vector> with_empty_group; + with_empty_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_group.emplace_back(); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_group), read_schema_, 1024, pool_), + "should not be empty"); + + // a gap segment must cover at least one selected row id + std::vector> with_empty_gap; + with_empty_gap.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_gap.push_back(MakeGroup({SegmentSpec::GapRanges({})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_gap), read_schema_, 1024, pool_), + "at least one selected row id"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 21d2b8db7..41426a2bf 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -208,11 +208,11 @@ AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetDataFileWriterFactory( AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const { + const std::optional>& write_cols, bool write_placeholder) const { std::shared_ptr path_factory = path_factory_; return std::make_shared(options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, - memory_pool_); + write_placeholder, memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( @@ -220,8 +220,13 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; + // A data-evolution write touching only blob columns is a partial update: its rows may mark + // untouched blobs with the placeholder sentinel (see BlobDefs::kWritePlaceholderKey). Any + // write that also carries non-blob columns stores blob bytes verbatim. + bool write_placeholder = + options_.DataEvolutionEnabled() && schemas.main_schema->num_fields() == 0; MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator = - [this, blob_schema](const std::string& blob_field_name) + [this, blob_schema, write_placeholder](const std::string& blob_field_name) -> Result< std::unique_ptr>>> { // Create a single-field schema for this blob field @@ -233,7 +238,7 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri auto single_field_schema = arrow::schema({field}); std::vector write_cols = {blob_field_name}; auto single_blob_file_writer_factory = - GetBlobFileWriterFactory(single_field_schema, write_cols); + GetBlobFileWriterFactory(single_field_schema, write_cols, write_placeholder); return std::make_unique>>( options_.GetBlobTargetFileSize(), single_blob_file_writer_factory); }; diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 4e64c0d99..066bd0259 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -110,7 +110,7 @@ class AppendOnlyWriter : public BatchWriter { WriterFactory GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const; + const std::optional>& write_cols, bool write_placeholder) const; Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp index aca03e7ab..86aa4a83f 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.cpp +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -17,8 +17,11 @@ #include "paimon/core/io/blob_data_file_writer_factory.h" #include +#include +#include #include +#include "paimon/common/data/blob_defs.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/manifest/file_source.h" @@ -33,20 +36,25 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory( const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, + const std::shared_ptr& path_factory, bool write_placeholder, const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), file_schema_(file_schema), write_cols_(write_cols), seq_num_counter_(seq_num_counter), - path_factory_(path_factory) {} + path_factory_(path_factory), + write_placeholder_(write_placeholder) {} Result>>> BlobDataFileWriterFactory::CreateWriter() const { std::shared_ptr seq_num_counter = options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::map format_options = options_.ToMap(); + if (write_placeholder_) { + format_options[BlobDefs::kWritePlaceholderKey] = "true"; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); + FileFormatFactory::Get("blob", format_options)); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema_, /*create_stats_extractor=*/true)); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h index e79d42e13..3ff813f97 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.h +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -43,12 +43,15 @@ class BlobDataFileWriterFactory : public DataFileWriterFactory, public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { public: + /// `write_placeholder` marks a data-evolution partial-update write: the blob format + /// writer is created with BlobDefs::kWritePlaceholderKey and persists placeholder + /// sentinel rows as placeholder entries. BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + bool write_placeholder, const std::shared_ptr& pool); Result>>> CreateWriter() const override; @@ -58,6 +61,7 @@ class BlobDataFileWriterFactory std::optional> write_cols_; std::shared_ptr seq_num_counter_; std::shared_ptr path_factory_; + bool write_placeholder_ = false; }; } // namespace paimon diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index 29b610b16..0c724d88a 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -39,9 +39,18 @@ CompleteRowTrackingFieldsBatchReader::CompleteRowTrackingFieldsBatchReader( Status CompleteRowTrackingFieldsBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, - arrow::ImportSchema(c_file_schema.get())); + // A format without a self-describing file schema (e.g. blob) cannot hold the special + // fields, so treat it as an empty file schema and synthesize them from the file meta. + std::shared_ptr file_schema; + Result> file_schema_result = reader_->GetFileSchema(); + if (file_schema_result.ok()) { + std::unique_ptr<::ArrowSchema> c_file_schema = std::move(file_schema_result).value(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(file_schema, arrow::ImportSchema(c_file_schema.get())); + } else if (file_schema_result.status().IsNotImplemented()) { + file_schema = arrow::schema(arrow::FieldVector{}); + } else { + return file_schema_result.status(); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); read_schema_ = arrow_schema; diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index ab323a235..4cea81edf 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -345,7 +345,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr> raw_readers, split_read_->CreateRawFileReaders(partition_, {file}, read_schema_, /*predicate=*/nullptr, dv_factory_, - /*row_ranges=*/std::nullopt, data_file_path_factory_)); + /*row_ranges=*/std::nullopt, data_file_path_factory_, + /*extra_format_options=*/{})); if (raw_readers.size() != 1) { return Status::Invalid("Unexpected, CreateSstFileFromDataFile only create single reader"); } diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index dc31aff23..9aa4ec921 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -75,7 +75,8 @@ Result>> AbstractSplitRead::CreateR const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const { + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const { if (data_files.empty()) { return std::vector>(); } @@ -89,7 +90,7 @@ Result>> AbstractSplitRead::CreateR auto data_file_path = data_file_path_factory->ToPath(file); PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, - PrepareReaderBuilder(data_file_identifier)); + PrepareReaderBuilder(data_file_identifier, extra_format_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_reader, CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(), @@ -120,9 +121,14 @@ Result> AbstractSplitRead::ApplyPredicateFilterIfNe } Result> AbstractSplitRead::PrepareReaderBuilder( - const std::string& format_identifier) const { + const std::string& format_identifier, + const std::map& extra_format_options) const { + std::map format_options = options_.ToMap(); + for (const auto& [key, value] : extra_format_options) { + format_options[key] = value; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format, - FileFormatFactory::Get(format_identifier, options_.ToMap())); + FileFormatFactory::Get(format_identifier, format_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, file_format->CreateReaderBuilder(options_.GetReadBatchSize())); reader_builder->WithMemoryPool(pool_); diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index 89c94fbf4..933c284d5 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -60,12 +61,16 @@ class AbstractSplitRead : public SplitRead { public: ~AbstractSplitRead() override = default; + /// `extra_format_options` are merged over the table options when building the format + /// reader, e.g. to switch the blob format reader into placeholder-aware mode for the + /// data-evolution blob fallback read path. Result>> CreateRawFileReaders( const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const; + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const; protected: AbstractSplitRead(const std::shared_ptr& path_factory, @@ -95,7 +100,8 @@ class AbstractSplitRead : public SplitRead { private: Result> PrepareReaderBuilder( - const std::string& format_identifier) const; + const std::string& format_identifier, + const std::map& extra_format_options) const; Result> CreateFileBatchReader( const std::string& file_format_identifier, const std::string& data_file_path, diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 1aecaa6d8..28017ccfc 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -16,6 +16,8 @@ #include "paimon/core/operation/data_evolution_split_read.h" +#include +#include #include #include #include @@ -28,10 +30,12 @@ #include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "paimon/common/catalog/catalog_context.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/global_index/complete_index_score_batch_reader.h" +#include "paimon/common/reader/blob_fallback_batch_reader.h" #include "paimon/common/reader/blob_view_resolving_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -45,73 +49,63 @@ #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/utils/blob_view_lookup.h" namespace paimon { +int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const { + if (files_.empty()) { + return 0; + } + if (!has_row_ids_selection_) { + // Add enforces the union range to be contiguous + return union_end_row_id_ - union_first_row_id_; + } + // with a row-ids selection the scan may have pruned files, leaving holes in the union + int64_t row_count = 0; + for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) { + row_count += range.Count(); + } + return row_count; +} + Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) { if (!BlobUtils::IsBlobFile(file->file_name)) { return Status::Invalid("Only blob file can be added to a blob bunch."); } PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); - if (first_row_id == latest_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with same first row id should have decreasing sequence number."); - } - // for files with the same first row id, file with larger sequence_number will be chosen, - // other files will be skipped - return Status::OK(); - } if (!files_.empty()) { - if (has_row_ids_selection_) { - // for the case: - // snapshot 1: blob0 [0, 9] - // snapshot 2: blob1 [0, 4] + blob2 [5, 9] - // when selected row id is {5}, only blob0 and blob2 is reserved in scan process, as - // blob1 has no intersect with {5} - // BlobBunch will first add blob0 [0, 9] - // then when it comes to blob2 [5, 9], blob0 will be removed as it has smaller sequence - // number - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number > latest_max_sequence_number_) { - row_count_ -= files_.back()->row_count; - files_.pop_back(); - } else { - return Status::OK(); - } - } - } else { - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with overlapping row id should have decreasing sequence " - "number."); - } - // for files with overlapping, if the file with smaller sequence_number is chosen, - // there will not exist file with larger sequence_number - return Status::OK(); - } else if (first_row_id > expected_next_first_row_id_) { - return Status::Invalid( - fmt::format("Blob file first row id should be continuous, expect {} but got {}", - expected_next_first_row_id_, first_row_id)); - } + if (file->schema_id != files_[0]->schema_id) { + return Status::Invalid("All files in a blob bunch should have the same schema id."); } - if (!files_.empty()) { - if (file->schema_id != files_[0]->schema_id) { - return Status::Invalid("All files in a blob bunch should have the same schema id."); - } - if (file->write_cols != files_[0]->write_cols) { - return Status::Invalid( - "All files in a blob bunch should have the same write columns."); - } + if (file->write_cols != files_[0]->write_cols) { + return Status::Invalid("All files in a blob bunch should have the same write columns."); } } - row_count_ += file->row_count; - if (row_count_ > expected_row_count_) { + // files sharing a max sequence number form one layer, whose row id ranges must be disjoint; + // overlaps across layers are the expected shape of partial updates and are kept for the + // row-level placeholder fallback + auto [layer_iter, layer_inserted] = + sequence_group_end_.try_emplace(file->max_sequence_number, 0); + if (!layer_inserted && first_row_id < layer_iter->second) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping row id " + "ranges: file {} (max sequence number {}) starts at row id {} before the previous " + "file's end {}", + file->file_name, file->max_sequence_number, first_row_id, layer_iter->second)); + } + if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) { + // a hole no layer covers cannot be aligned with the data files return Status::Invalid( - fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + fmt::format("Blob file first row id should be continuous, expect {} but got {}", + union_end_row_id_, first_row_id)); } + int64_t end_row_id = first_row_id + file->row_count; + layer_iter->second = end_row_id; + union_first_row_id_ = std::min(union_first_row_id_, first_row_id); + union_end_row_id_ = std::max(union_end_row_id_, end_row_id); + ranges_.emplace_back(first_row_id, end_row_id - 1); files_.push_back(file); - latest_max_sequence_number_ = file->max_sequence_number; - latest_first_row_id_ = first_row_id; - expected_next_first_row_id_ = latest_first_row_id_ + file->row_count; + if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) { + return Status::Invalid( + fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + } return Status::OK(); } DataEvolutionSplitRead::DataEvolutionSplitRead( @@ -235,7 +229,8 @@ Result> DataEvolutionSplitRead::CreateBlobViewReade std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema, /*predicate=*/nullptr, /*dv_factory=*/nullptr, - /*row_ranges=*/std::nullopt, data_file_path_factory)); + /*row_ranges=*/std::nullopt, data_file_path_factory, + /*extra_format_options=*/{})); auto batch_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -314,7 +309,8 @@ Result> DataEvolutionSplitRead::InnerCreateReader( std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_, /*predicate=*/nullptr, - /*dv_factory=*/nullptr, row_ranges, data_file_path_factory)); + /*dv_factory=*/nullptr, row_ranges, data_file_path_factory, + /*extra_format_options=*/{})); assert(raw_file_readers.size() == 1); sub_readers.push_back(std::move(raw_file_readers[0])); } else { @@ -490,18 +486,30 @@ Result> DataEvolutionSplitRead::CreateU if (!read_fields_in_file.empty()) { // create new FieldMappingReader for read partial fields auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file); - PAIMON_ASSIGN_OR_RAISE(std::vector> file_readers, - CreateRawFileReaders(partition, bunch->Files(), file_read_schema, - /*predicate=*/nullptr, /*dv_factory=*/{}, - row_ranges, data_file_path_factory)); - if (file_readers.size() == 1) { - file_batch_readers[file_idx] = std::move(file_readers[0]); + auto blob_bunch = std::dynamic_pointer_cast(bunch); + if (blob_bunch && !blob_bunch->SequentialReadOptimize()) { + // blob files span multiple max sequence number layers: placeholder entries of + // newer layers must fall back row by row to older layers + PAIMON_ASSIGN_OR_RAISE( + file_batch_readers[file_idx], + CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema, + row_ranges, data_file_path_factory)); } else { - auto raw_readers = - ObjectUtils::MoveVector>(std::move(file_readers)); - // Concat multiple blob files that map to the same data file. - file_batch_readers[file_idx] = - std::make_unique(std::move(raw_readers), pool_); + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, bunch->Files(), file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, + /*extra_format_options=*/{})); + if (file_readers.size() == 1) { + file_batch_readers[file_idx] = std::move(file_readers[0]); + } else { + auto raw_readers = ObjectUtils::MoveVector>( + std::move(file_readers)); + // Concat multiple blob files that map to the same data file. + file_batch_readers[file_idx] = + std::make_unique(std::move(raw_readers), pool_); + } } } } @@ -511,6 +519,98 @@ Result> DataEvolutionSplitRead::CreateU field_offsets, pool_); } +namespace { +/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a +/// selection, otherwise its intersection with the (possibly overlapping) selected ranges. +std::vector SelectedRangesInRange(int64_t from, int64_t to, + const std::optional>& row_ranges) { + if (!row_ranges) { + return {Range(from, to)}; + } + std::vector selected; + Range gap_range(from, to); + for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) { + std::optional intersection = Range::Intersection(gap_range, range); + if (intersection) { + selected.push_back(*intersection); + } + } + return selected; +} +} // namespace + +Result> DataEvolutionSplitRead::CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const { + int64_t union_first_row_id = std::numeric_limits::max(); + int64_t union_last_row_id = std::numeric_limits::min(); + using FileWithFirstRowId = std::pair>; + std::map, std::greater<>> sequence_groups; + for (const auto& file : files) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); + union_first_row_id = std::min(union_first_row_id, first_row_id); + union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1); + sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file); + } + // the blob format reader must emit placeholder sentinels instead of failing on them + const std::map blob_format_options = { + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + std::vector> groups; + groups.reserve(sequence_groups.size()); + for (auto& [max_sequence_number, group_files] : sequence_groups) { + std::stable_sort(group_files.begin(), group_files.end(), + [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) { + return f1.first < f2.first; + }); + std::vector segments; + // pad row ids this layer does not cover with placeholder gaps, so that every layer spans + // the same union range and the groups can be stepped in lockstep + int64_t next_row_id = union_first_row_id; + for (const auto& [first_row_id, file] : group_files) { + if (first_row_id < next_row_id) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping " + "row id ranges: file {} (max sequence number {}) starts at row id {} before " + "the previous file's end {}", + file->file_name, max_sequence_number, first_row_id, next_row_id)); + } + if (first_row_id > next_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, {file}, file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, blob_format_options)); + if (file_readers.size() != 1) { + return Status::Invalid("Unexpected: blob fallback file reader was skipped."); + } + segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}}); + next_row_id = first_row_id + file->row_count; + } + if (next_row_id <= union_last_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + groups.push_back(std::move(segments)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_reader, + BlobFallbackBatchReader::Create(std::move(groups), file_read_schema, + options_.GetReadBatchSize(), pool_)); + return std::move(fallback_reader); +} + Result DataEvolutionSplitRead::Match(const std::shared_ptr& split, bool force_keep_delete) const { return true; diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index c3ed5033f..0ca800604 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -107,26 +109,40 @@ class DataEvolutionSplitRead : public AbstractSplitRead { std::vector> data_files_; }; + /// All blob files of one blob field in a merge group. Unlike data files, every file is kept + /// (aligned with Java's BlobFileBunch): files whose max sequence numbers differ form layers + /// of a data-evolution partial update, where newer layers record untouched rows as + /// placeholder entries and reading falls back row by row to older layers. + /// Files must be added ordered by first row id ascending (then max sequence number + /// descending), as produced by MergeRangesAndSort. class BlobBunch : public FieldBunch { public: explicit BlobBunch(int64_t expected_row_count, bool has_row_ids_selection) : expected_row_count_(expected_row_count), has_row_ids_selection_(has_row_ids_selection) {} - int64_t RowCount() const override { - return row_count_; - } + /// Number of distinct row ids covered by the added files. Without a row-ids selection + /// the covered range is contiguous (enforced by Add) and matches the data files. + int64_t RowCount() const override; const std::vector>& Files() const override { return files_; } Status Add(const std::shared_ptr& file); + /// True when every file shares one max sequence number: no placeholder entry can need + /// resolving, so the files can be read sequentially without the fallback merge. + bool SequentialReadOptimize() const { + return sequence_group_end_.size() <= 1; + } private: int64_t expected_row_count_ = -1; - int64_t latest_first_row_id_ = -1; - int64_t expected_next_first_row_id_ = -1; - int64_t latest_max_sequence_number_ = -1; - int64_t row_count_ = 0; bool has_row_ids_selection_ = false; + int64_t union_first_row_id_ = std::numeric_limits::max(); + /// Exclusive end of the union row id range covered so far. + int64_t union_end_row_id_ = std::numeric_limits::min(); + /// Per max sequence number: exclusive end of the last added range, to reject + /// overlapping files within one layer. + std::map sequence_group_end_; + std::vector ranges_; std::vector> files_; }; @@ -164,6 +180,16 @@ class DataEvolutionSplitRead : public AbstractSplitRead { const std::vector>& need_merge_files, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; + + /// Builds the row-level fallback reader for a blob bunch spanning multiple max sequence + /// number layers: groups the files by max sequence number, pads uncovered row id ranges of + /// each layer with placeholder gap segments, and resolves each row to the newest + /// non-placeholder layer. See BlobFallbackBatchReader. + Result> CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const; }; } // namespace paimon diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index aa6f9c9e1..4cf3a664b 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -145,72 +145,88 @@ TEST_F(DataEvolutionSplitReadTest, TestAddNonBlobFileInvalid) { TEST_F(DataEvolutionSplitReadTest, TestAddBlobWithSameFirstRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/1, + /*max_sequence_number=*/3, /*write_cols=*/std::optional>({"blob_col"})); - auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + auto blob_full_tail = + CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); + auto blob_short_tail = + CreateBlobFile("blob3", /*first_row_id=*/0, /*row_count=*/50, + /*max_sequence_number=*/1, + /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), - "Blob file with same first row id should have decreasing sequence number."); + // Files with the same first row id and lower sequence numbers are older layers of a + // partial update; they are kept for the row-level placeholder fallback, whether they + // cover the same range or only a shorter prefix of it. + ASSERT_OK(blob_bunch->Add(blob_full_tail)); + ASSERT_OK(blob_bunch->Add(blob_short_tail)); + + ASSERT_EQ(blob_bunch->Files().size(), 3); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_full_tail); + ASSERT_EQ(blob_bunch->Files()[2], blob_short_tail); + ASSERT_EQ(blob_bunch->RowCount(), 100); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithSameFirstRowIdAndLowerSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with same firstRowId and lower sequence number should be ignored + // Overlapping layers with different sequence numbers are kept for the fallback. ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/1, + /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with overlapping row id and lower sequence number should be ignored ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdInSameLayer) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG( - blob_bunch->Add(blob_tail), - "Blob file with overlapping row id should have decreasing sequence number."); + // Files sharing a max sequence number form one layer and must not overlap. + ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), + "Blob files with the same max sequence number should not have overlapping " + "row id ranges"); } TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithNonContinuousRowId) { @@ -265,12 +281,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, blob_entry is removed + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is a newer + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_sub2); - ASSERT_EQ(blob_bunch->RowCount(), 5); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); + ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { @@ -291,13 +309,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, as blob_sub2 has smaller sequence number, blob_sub2 will be - // skipped. + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is an older + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); + ASSERT_EQ(blob_bunch->Files().size(), 2); ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelection) { @@ -420,15 +439,15 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { std::vector> batch = batches[0]; ASSERT_EQ(batch.size(), 10); ASSERT_EQ(batch[0], data); - ASSERT_EQ(batch[1], blob_entry5); // pick - ASSERT_EQ(batch[2], blob_entry2); // skip - ASSERT_EQ(batch[3], blob_entry1); // skip - ASSERT_EQ(batch[4], blob_entry9); // pick - ASSERT_EQ(batch[5], blob_entry6); // skip - ASSERT_EQ(batch[6], blob_entry3); // skip - ASSERT_EQ(batch[7], blob_entry7); // pick - ASSERT_EQ(batch[8], blob_entry4); // skip - ASSERT_EQ(batch[9], blob_entry8); // pick + ASSERT_EQ(batch[1], blob_entry5); + ASSERT_EQ(batch[2], blob_entry2); + ASSERT_EQ(batch[3], blob_entry1); + ASSERT_EQ(batch[4], blob_entry9); + ASSERT_EQ(batch[5], blob_entry6); + ASSERT_EQ(batch[6], blob_entry3); + ASSERT_EQ(batch[7], blob_entry7); + ASSERT_EQ(batch[8], blob_entry4); + ASSERT_EQ(batch[9], blob_entry8); auto blob_field_to_field_id = [](const std::shared_ptr&) -> Result { return 0; @@ -440,12 +459,19 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { ASSERT_EQ(bunch.size(), 2); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { @@ -561,20 +587,33 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { ASSERT_EQ(bunch.size(), 3); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); auto blob_bunch2 = std::dynamic_pointer_cast(bunch[2]); - ASSERT_EQ(blob_bunch2->Files().size(), 4); + ASSERT_EQ(blob_bunch2->Files().size(), 9); ASSERT_EQ(blob_bunch2->Files()[0], blob_entry15); - ASSERT_EQ(blob_bunch2->Files()[1], blob_entry19); - ASSERT_EQ(blob_bunch2->Files()[2], blob_entry17); - ASSERT_EQ(blob_bunch2->Files()[3], blob_entry18); + ASSERT_EQ(blob_bunch2->Files()[1], blob_entry12); + ASSERT_EQ(blob_bunch2->Files()[2], blob_entry11); + ASSERT_EQ(blob_bunch2->Files()[3], blob_entry19); + ASSERT_EQ(blob_bunch2->Files()[4], blob_entry16); + ASSERT_EQ(blob_bunch2->Files()[5], blob_entry13); + ASSERT_EQ(blob_bunch2->Files()[6], blob_entry17); + ASSERT_EQ(blob_bunch2->Files()[7], blob_entry14); + ASSERT_EQ(blob_bunch2->Files()[8], blob_entry18); ASSERT_EQ(blob_bunch2->RowCount(), 1000); + ASSERT_FALSE(blob_bunch2->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestDifferentRowIdRange) { diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index 86df4841d..e1cee5a08 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -272,7 +272,8 @@ Result> MergeFileSplitRead::CreateNoMergeReader( std::vector> raw_file_readers, CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema, only_filter_key ? predicate_for_keys_ : context_->GetPredicate(), - dv_factory, /*row_ranges=*/{}, data_file_path_factory)); + dv_factory, /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -496,7 +497,8 @@ Result> MergeFileSplitRead::CreateReaderFo PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); assert(data_files.size() == raw_file_readers.size()); // KeyValueDataFileRecordReader converts arrow array from format reader to KeyValue objects diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index ff213a4ea..c027e1c72 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -81,7 +81,8 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index d5b5a2fe2..feb309cdd 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -18,6 +18,7 @@ #include #include +#include #include "arrow/api.h" #include "arrow/array/builder_dict.h" @@ -38,7 +39,7 @@ namespace paimon::blob { Result> BlobFileBatchReader::Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, - const std::shared_ptr& pool) { + bool emit_placeholder_sentinel, const std::shared_ptr& pool) { if (input_stream == nullptr) { return Status::Invalid("blob file batch reader create failed: input stream is nullptr"); } @@ -89,8 +90,9 @@ Result> BlobFileBatchReader::Create( } } PAIMON_ASSIGN_OR_RAISE(std::string file_path, input_stream->GetUri()); - auto reader = std::unique_ptr(new BlobFileBatchReader( - input_stream, file_path, blob_lengths, blob_offsets, batch_size, blob_as_descriptor, pool)); + auto reader = std::unique_ptr( + new BlobFileBatchReader(input_stream, file_path, blob_lengths, blob_offsets, batch_size, + blob_as_descriptor, emit_placeholder_sentinel, pool)); return reader; } @@ -99,6 +101,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, bool blob_as_descriptor, + bool emit_placeholder_sentinel, const std::shared_ptr& pool) : input_stream_(input_stream), file_path_(file_path), @@ -108,6 +111,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp target_blob_offsets_(blob_offsets), batch_size_(batch_size), blob_as_descriptor_(blob_as_descriptor), + emit_placeholder_sentinel_(emit_placeholder_sentinel), pool_(pool), arrow_pool_(GetArrowPool(pool_)), metrics_(std::make_shared()) { @@ -168,11 +172,8 @@ Result> BlobFileBatchReader::NextBlobOffsets( PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(0)); int64_t data_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; // Null blobs contribute zero bytes to content - if (!IsTargetNull(i)) { - data_length += GetTargetContentLength(i); - } + data_length += GetTargetOutputLength(current_pos_ + k); PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(data_length)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offset_buffer, @@ -184,10 +185,7 @@ Result> BlobFileBatchReader::NextBlobContents( int32_t rows_to_read) const { int64_t total_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - if (!IsTargetNull(i)) { - total_length += GetTargetContentLength(i); - } + total_length += GetTargetOutputLength(current_pos_ + k); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr data_buffer, arrow::AllocateBuffer(total_length, arrow_pool_.get())); @@ -205,6 +203,46 @@ Result> BlobFileBatchReader::NextBlobContents( return data_buffer; } +Result> BlobFileBatchReader::BuildSentinelAwareContentArray( + int32_t rows_to_read) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr array_builder, + arrow::MakeBuilder(target_type_, arrow_pool_.get())); + auto builder = dynamic_cast(array_builder.get()); + if (builder == nullptr) { + return Status::Invalid("cast to struct builder failed"); + } + auto field_builder = dynamic_cast(builder->field_builder(0)); + if (field_builder == nullptr) { + return Status::Invalid("cast to large binary builder failed"); + } + std::string content; + for (int32_t k = 0; k < rows_to_read; ++k) { + const size_t i = current_pos_ + k; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); + if (IsTargetNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + continue; + } + if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); + continue; + } + int64_t length = GetTargetContentLength(i); + content.resize(length); + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(GetTargetContentOffset(i), length, + reinterpret_cast(content.data()))); + if (BlobDefs::HasPlaceholderSentinelPrefix(content.data(), content.size())) { + // escape so the fallback merge cannot mistake the value for a placeholder + content.insert(0, BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append(content.data(), content.size())); + } + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array)); + return array; +} + Result> BlobFileBatchReader::BuildNullBitmap( int32_t rows_to_read) const { bool has_null = false; @@ -250,6 +288,9 @@ Result> BlobFileBatchReader::BuildTargetArray( int32_t rows_to_read) const { std::shared_ptr blob_array; if (!blob_as_descriptor_) { + if (emit_placeholder_sentinel_) { + return BuildSentinelAwareContentArray(rows_to_read); + } return BuildContentArray(rows_to_read); } // For descriptor mode, build using StructBuilder to handle nulls properly @@ -268,6 +309,9 @@ Result> BlobFileBatchReader::BuildTargetArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); if (IsTargetNull(i)) { PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + } else if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); } else { int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); @@ -297,6 +341,17 @@ Result BlobFileBatchReader::NextBatch() { } int32_t left_rows = target_blob_lengths_.size() - current_pos_; int32_t rows_to_read = std::min(left_rows, batch_size_); + if (!emit_placeholder_sentinel_) { + for (int32_t k = 0; k < rows_to_read; ++k) { + if (IsTargetPlaceholder(current_pos_ + k)) { + return Status::Invalid(fmt::format( + "blob file {} contains a placeholder entry (bin_length {}) written by a " + "data-evolution partial update; it can only be resolved by the data-evolution " + "blob fallback read path", + file_path_, BlobDefs::kPlaceholderBinLength)); + } + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob_array, BuildTargetArray(rows_to_read)); std::unique_ptr c_array = std::make_unique(); diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 27549c190..a01bf42c8 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -87,9 +87,17 @@ namespace paimon::blob { /// - Current version is 1. class BlobFileBatchReader : public FileBatchReader { public: + /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length == + /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving + /// them requires the data-evolution blob fallback path; when true they are returned as the + /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away, and any + /// stored value starting with the sentinel bytes is escaped by prepending one sentinel + /// copy, so real values can never be mistaken for placeholders (the fallback merge strips + /// the escape). static Result> Create( const std::shared_ptr& input_stream, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Result> GetFileSchema() const override; @@ -144,7 +152,8 @@ class BlobFileBatchReader : public FileBatchReader { BlobFileBatchReader(const std::shared_ptr& input_stream, const std::string& file_path, const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Status ReadBlobContentAt(const int64_t offset, const int64_t length, uint8_t* content) const; @@ -153,6 +162,11 @@ class BlobFileBatchReader : public FileBatchReader { /// Builds a null bitmap buffer for the given rows. Returns nullptr if no nulls. Result> BuildNullBitmap(int32_t rows_to_read) const; Result> BuildContentArray(int32_t rows_to_read) const; + /// Raw-content variant of BuildContentArray for placeholder-aware reads: placeholder + /// entries become the sentinel bytes and stored values starting with the sentinel are + /// escaped by prepending one sentinel copy. + Result> BuildSentinelAwareContentArray( + int32_t rows_to_read) const; Result> BuildTargetArray(int32_t rows_to_read) const; /// Returns true if the blob at the given index is null (bin_length == kNullBinLength). @@ -160,6 +174,20 @@ class BlobFileBatchReader : public FileBatchReader { return target_blob_lengths_[index] == BlobDefs::kNullBinLength; } + bool IsTargetPlaceholder(size_t index) const { + return target_blob_lengths_[index] == BlobDefs::kPlaceholderBinLength; + } + + /// Content bytes the blob at the given index contributes to the output buffer. Only used + /// by the fast content path, which never sees placeholder entries: strict mode fails on + /// them first and placeholder-aware mode goes through BuildSentinelAwareContentArray. + int64_t GetTargetOutputLength(size_t index) const { + if (IsTargetNull(index)) { + return 0; + } + return GetTargetContentLength(index); + } + int64_t GetTargetContentOffset(size_t index) const { return target_blob_offsets_[index] + BlobDefs::kContentStartOffset; } @@ -179,6 +207,7 @@ class BlobFileBatchReader : public FileBatchReader { const int32_t batch_size_; const bool blob_as_descriptor_; + const bool emit_placeholder_sentinel_; std::shared_ptr pool_; std::shared_ptr arrow_pool_; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 4e53dfb4c..ca524431d 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -162,9 +162,10 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_OK_AND_ASSIGN( std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -213,9 +214,9 @@ TEST_F(BlobFileBatchReaderTest, InvalidScenario) { } { ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(/*input_stream=*/input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(/*input_stream=*/input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->GetFileSchema(), "blob file has no self-describing file schema"); ASSERT_TRUE(reader->GetReaderMetrics()); @@ -237,7 +238,8 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -248,9 +250,10 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(dir->Str() + "/file.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -273,9 +276,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(/*read_schema=*/nullptr, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "SetReadSchema failed: read schema cannot be nullptr"); @@ -298,9 +301,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "read schema field number 2 is not 1"); @@ -323,9 +326,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "field my_blob_field: large_binary is not BLOB"); @@ -346,9 +349,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); RoaringBitmap32 roaring; roaring.Add(0); roaring.Add(1); diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index 875566ffc..d1dd48812 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -39,7 +39,7 @@ namespace paimon::blob { BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, - bool write_null_on_fetch_failure, + bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) : out_(out), @@ -48,7 +48,8 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con fs_(fs), pool_(pool), write_null_on_missing_file_(write_null_on_missing_file), - write_null_on_fetch_failure_(write_null_on_fetch_failure) { + write_null_on_fetch_failure_(write_null_on_fetch_failure), + write_placeholder_(write_placeholder) { // Create() has already checked that data_type has exactly one BLOB field. blob_field_name_ = data_type_->field(0)->name(); metrics_ = std::make_shared(); @@ -59,7 +60,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con Result> BlobFormatWriter::Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); @@ -82,8 +83,9 @@ Result> BlobFormatWriter::Create( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); - return std::unique_ptr(new BlobFormatWriter( - out, uri, data_type, write_null_on_missing_file, write_null_on_fetch_failure, fs, pool)); + return std::unique_ptr( + new BlobFormatWriter(out, uri, data_type, write_null_on_missing_file, + write_null_on_fetch_failure, write_placeholder, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -117,7 +119,20 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { const auto& blob_array = arrow::internal::checked_cast(*child_array); assert(blob_array.length() == 1); - PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); + std::string_view blob_data = blob_array.GetView(0); + // Only a data-evolution partial-update write interprets the placeholder protocol; any + // other write stores the bytes verbatim, so user data can never become a placeholder. + if (write_placeholder_ && + BlobDefs::HasPlaceholderSentinelPrefix(blob_data.data(), blob_data.size())) { + if (BlobDefs::IsPlaceholderSentinel(blob_data.data(), blob_data.size())) { + // row not touched by the partial update: record it as a placeholder entry + bin_lengths_.push_back(BlobDefs::kPlaceholderBinLength); + return Status::OK(); + } + // a real value escaped by the producer: strip the leading sentinel copy + blob_data.remove_prefix(BlobDefs::kPlaceholderSentinelLength); + } + PAIMON_RETURN_NOT_OK(WriteBlob(blob_data)); PAIMON_RETURN_NOT_OK(Flush()); return Status::OK(); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 9dbe57a87..615d64c47 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -68,9 +68,16 @@ class BlobFormatWriter : public FormatWriter { /// `write_null_on_missing_file` is enabled; otherwise a missing file follows /// `write_null_on_fetch_failure` like any other failed open. /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. + /// + /// `write_placeholder` (see BlobDefs::kWritePlaceholderKey, false unless the write is a + /// data-evolution partial update) applies the placeholder protocol to incoming values: a + /// value equal to BlobDefs::kPlaceholderSentinel is persisted as a placeholder entry + /// (bin_length -2, no data bytes), and a longer value starting with the sentinel is + /// unescaped by stripping the leading sentinel before writing. When disabled, values are + /// stored verbatim and can never be turned into placeholder entries. static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -91,7 +98,7 @@ class BlobFormatWriter : public FormatWriter { BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, bool write_null_on_fetch_failure, - const std::shared_ptr& fs, + bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); @@ -135,6 +142,7 @@ class BlobFormatWriter : public FormatWriter { std::shared_ptr metrics_; bool write_null_on_missing_file_ = false; bool write_null_on_fetch_failure_ = false; + bool write_placeholder_ = false; uint64_t null_on_missing_file_count_ = 0; uint64_t null_on_fetch_failure_count_ = 0; std::unique_ptr logger_; diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index d54f6042c..d067a7624 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -129,7 +129,16 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> CreateDefaultWriter() const { return BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_); + } + + /// Create a writer in placeholder mode, as used by data-evolution partial updates. + Result> CreatePlaceholderWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/true, file_system_, pool_); } Status AddBatchOnce(const std::shared_ptr& format_writer, @@ -161,9 +170,11 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> ReadBackAsData() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - /*blob_as_descriptor=*/false, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); @@ -253,7 +264,8 @@ TEST_P(BlobFormatWriterTest, TestSimple) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -284,41 +296,47 @@ TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. out is nullptr"); // Test with nullptr data type ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, nullptr), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); // Test with nullptr file system ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, nullptr, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, nullptr, pool_), "blob format writer create failed. fs is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, multi_field_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, non_blob_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "field regular_col: binary is not BLOB"); } @@ -443,7 +461,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -492,7 +511,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -525,7 +545,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, @@ -563,7 +584,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); std::string file = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, @@ -602,7 +624,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); // Row 0: missing file -> NULL. ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, @@ -667,7 +690,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), 0); } @@ -676,7 +700,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); } // The same fetch failure, now converted to NULL. @@ -684,7 +709,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); } // Missing file with only fetch-failure enabled: no existence check runs, so the file is @@ -695,7 +721,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1); ASSERT_OK_AND_ASSIGN( @@ -713,7 +740,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); // One check before the open and one after it. ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); @@ -733,7 +761,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); ASSERT_OK_AND_ASSIGN( @@ -752,7 +781,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -762,7 +792,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -785,14 +816,16 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor"); } { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -820,7 +853,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1); ASSERT_OK(writer->Flush()); @@ -850,7 +884,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, exists_fail_fs, pool_)); // The reported failure names the check and keeps the underlying status message. Status check_status = AddBatchOnce(writer, array); ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file"); @@ -864,7 +899,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); // One check before the open and one after it failed. ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2); @@ -884,7 +920,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( side_stream, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0); ASSERT_OK_AND_ASSIGN( @@ -931,4 +968,264 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_EQ(buffer, expected); } +/// Placeholder tests always feed the sentinel bytes of the placeholder write protocol, so +/// they do not depend on the blob_as_descriptor_ parameter and run once on the +/// non-parameterized fixture. +using BlobFormatWriterPlaceholderTest = BlobFormatWriterTestBase; + +std::string PlaceholderSentinelBytes() { + return std::string(BlobDefs::PlaceholderSentinelView()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestWritePlaceholderGoldenBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + + // row 0: inline bytes "inline"; row 1: null; row 2: placeholder + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->AppendNull().ok()); + std::shared_ptr null_array; + ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); + ASSERT_OK(AddBatchOnce(writer, null_array)); + + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // Verify byte-level alignment with the Java writer: null and placeholder rows occupy no + // data bytes; the index records [22, -1, -2] as zigzag varint deltas [0x2c, 0x2d, 0x01]. + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_TRUE(input_stream); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); + ASSERT_EQ(file_length, 30); + std::vector buffer(file_length); + ASSERT_OK_AND_ASSIGN(auto read_length, + input_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_EQ(read_length, 30); + std::vector expected = {{// record 0: magic + "inline" + bin_length(22) + crc32 + 0xcf, 0x11, 0x4e, 0x58, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x60, + 0xc8, 0xe9, + // index of [22, -1, -2] + 0x2c, 0x2d, 0x01, + // footer: index length + version + 0x03, 0x00, 0x00, 0x00, 0x01}}; + ASSERT_EQ(buffer, expected); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderStrictAndAwareModes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // default (strict) mode: reading a placeholder entry fails + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "placeholder"); + } + + // placeholder-aware mode: the entry is returned as the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), "inline"); + ASSERT_FALSE(binary_array->IsNull(1)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } + + // placeholder-aware descriptor mode also returns the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderWithSelectionBitmap) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto array0, MakeBlobArrayFromBytes("first")); + ASSERT_OK(AddBatchOnce(writer, array0)); + ASSERT_OK_AND_ASSIGN(auto array1, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, array1)); + ASSERT_OK_AND_ASSIGN(auto array2, MakeBlobArrayFromBytes("third")); + ASSERT_OK(AddBatchOnce(writer, array2)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + RoaringBitmap32 selection; + selection.Add(1); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, selection)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); + ASSERT_EQ(binary_array->GetString(1), "third"); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelBytesVerbatimWithoutPlaceholderMode) { + // outside placeholder mode a user blob whose bytes equal the sentinel is a normal value: it + // must be stored as a real entry (not persisted as bin_length -2) and read back unchanged + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); + ASSERT_OK_AND_ASSIGN(auto sentinel_array, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, sentinel_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 1); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(binary_array->IsNull(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestEscapedSentinelRoundTrip) { + // in placeholder mode a real value starting with the sentinel must be escaped by the + // producer (one extra sentinel copy); the writer strips the escape and stores the original + // bytes, a strict read returns them verbatim, and a placeholder-aware read re-escapes them + // so the fallback merge cannot mistake them for placeholders + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + std::string sentinel = PlaceholderSentinelBytes(); + ASSERT_OK_AND_ASSIGN(auto escaped_sentinel_array, MakeBlobArrayFromBytes(sentinel + sentinel)); + ASSERT_OK(AddBatchOnce(writer, escaped_sentinel_array)); + ASSERT_OK_AND_ASSIGN(auto escaped_prefixed_array, + MakeBlobArrayFromBytes(sentinel + sentinel + "suffix")); + ASSERT_OK(AddBatchOnce(writer, escaped_prefixed_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // strict mode returns the original (unescaped) bytes verbatim + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + "suffix"); + } + + // placeholder-aware mode re-escapes the stored values with one extra sentinel copy + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), sentinel + sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + sentinel + "suffix"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_reader_builder.h b/src/paimon/format/blob/blob_reader_builder.h index eb2ce9522..a5d20fbdd 100644 --- a/src/paimon/format/blob/blob_reader_builder.h +++ b/src/paimon/format/blob/blob_reader_builder.h @@ -43,7 +43,11 @@ class BlobReaderBuilder : public ReaderBuilder { PAIMON_ASSIGN_OR_RAISE( bool blob_as_descriptor, OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, pool_); + PAIMON_ASSIGN_OR_RAISE(bool emit_placeholder_sentinel, + OptionsUtils::GetValueFromMap( + options_, BlobDefs::kEmitPlaceholderSentinelKey, false)); + return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, + emit_placeholder_sentinel, pool_); } private: diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp index ca5efe831..14445dfc6 100644 --- a/src/paimon/format/blob/blob_stats_extractor.cpp +++ b/src/paimon/format/blob/blob_stats_extractor.cpp @@ -51,7 +51,8 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_ PAIMON_ASSIGN_OR_RAISE( std::unique_ptr blob_reader, BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool)); + /*batch_size=*/1024, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool)); ColumnStatsVector result_stats; result_stats.push_back( ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt)); diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 12caa0e16..7cc0b5927 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -24,6 +24,7 @@ #include #include "arrow/api.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/utils/options_utils.h" #include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" @@ -73,8 +74,11 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure, OptionsUtils::GetValueFromMap( options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false)); + PAIMON_ASSIGN_OR_RAISE( + bool write_placeholder, + OptionsUtils::GetValueFromMap(options_, BlobDefs::kWritePlaceholderKey, false)); return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file, - write_null_on_fetch_failure, fs_, pool_); + write_null_on_fetch_failure, write_placeholder, fs_, pool_); } private: diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 3a8900099..590184632 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -42,6 +42,7 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" @@ -1045,6 +1046,486 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); } +/// Build a single-blob-column StructArray for a data-evolution partial update: "PH" marks a row +/// whose blob is not updated (persisted as a placeholder entry), std::nullopt a null blob. +std::shared_ptr MakeBlobUpdateArray( + const std::shared_ptr& blob_field, + const std::vector>& rows) { + auto struct_type = arrow::struct_({blob_field}); + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string_view sentinel = BlobDefs::PlaceholderSentinelView(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return std::dynamic_pointer_cast(array); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { + // the blob column is updated to null below, so it must be nullable + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Initial full-row write assigns row ids 0-2. + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "blob_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // Partial update: only row 1 gets a new blob, the untouched rows are written as + // placeholder entries and must fall back to the previous blob file when read. + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // updating a blob to null is not a placeholder: the null must win over older layers + auto null_update_array = MakeBlobUpdateArray(fields[2], {std::nullopt, "PH", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {}, {"b0"}, {null_update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array2)); + + // row ids still come from the data files and stay aligned with the fallback result + auto expected_with_row_id = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}), + R"([ + [1, "a", null, 0], + [2, "b", "updated_b", 1], + [3, "c", "blob_c", 2] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); + + // blob_as_descriptor read mode: the fallback-merged values come back as descriptors and + // must still resolve to the same bytes + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto desc_result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(desc_result.chunked_array); + auto desc_concat = arrow::Concatenate(desc_result.chunked_array->chunks()).ValueOrDie(); + auto desc_struct = std::dynamic_pointer_cast(desc_concat); + ASSERT_TRUE(desc_struct); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(desc_struct, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array2)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << "\nexpected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "blob_3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // second layer updates row 0 within rows [0, 1] + auto update_array1 = MakeBlobUpdateArray(fields[2], {"update1_0", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // third layer updates row 3 within rows [2, 3]; each layer only partially covers the + // range, the uncovered parts behave as placeholders + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "update2_3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "update1_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "update2_3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // row-range pushdown where the selected rows fall in one layer's file and in another + // layer's uncovered gap: row 1 is a gap row for the third layer, row 2 for the second + auto expected_middle = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, "b", "blob_1"], + [3, "c", "blob_2"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_middle, + /*predicate=*/nullptr, /*row_ranges=*/{Range(1, 2)})); + + // per-row reads resolve every row independently + const std::vector expected_rows = { + R"([[1, "a", "update1_0"]])", R"([[2, "b", "blob_1"]])", R"([[3, "c", "blob_2"]])", + R"([[4, "d", "update2_3"]])"}; + for (int32_t i = 0; i < static_cast(expected_rows.size()); i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_rows[i]) + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Mirrors the layer shapes of Java's BlobUpdateTest.testReadCompactedBlobSequenceGroups + // (with a single base write, as the C++ write path assigns one sequence per commit): + // row id: 0 1 2 3 4 5 6 7 8 9 + // seq1: [b0 b1 b2 b3 b4 b5 b6 b7 b8 b9] + // seq2: [u20 P P u23 u24] . . . . . + // seq3: . . . . . [P u46 P u48 P] + // seq4: [P u61 P P P P P P P u69] + // result: u20 u61 b2 u23 u24 b5 u46 b7 u48 u69 + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array1 = MakeBlobUpdateArray(fields[2], {"u20", "PH", "PH", "u23", "u24"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "u46", "PH", "u48", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/5, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto update_array3 = MakeBlobUpdateArray( + fields[2], {"PH", "u61", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "u69"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {}, {"b0"}, {update_array3})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs3); + ASSERT_OK(Commit(table_path, commit_msgs3)); + + const std::vector expected_blobs = {"u20", "u61", "blob_2", "u23", "u24", + "blob_5", "u46", "blob_7", "u48", "u69"}; + auto expected_row = [&](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"" + expected_blobs[i] + + "\""; + }; + auto expected_full = PrepareBulkData(10, expected_row, fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); + + // every row resolves independently under row-range pushdown + for (int32_t i = 0; i < 10; i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[" + expected_row(i) + "]]") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // partial update touching rows 1 and 9 only + auto update_array = MakeBlobUpdateArray( + fields[2], {"PH", "update_1", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "update_9"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // push down row ranges hitting updated and untouched rows + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "name_1", "update_1"], + [5, "name_5", "blob_5"], + [9, "name_9", "update_9"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 1), Range(5, 5), Range(9, 9)})); + + // full read still resolves every row + auto expected_full = PrepareBulkData( + 10, + [](int32_t i) { + std::string blob = (i == 1 || i == 9) ? "\"update_" + std::to_string(i) + "\"" + : "\"blob_" + std::to_string(i) + "\""; + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", " + blob; + }, + fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); +} + +/// A BLOB sequence layer whose physical file covers only a strict subrange of the full row id +/// range: the newest group is internally Gap(2), File([u2, u3]), Gap(6). On a BLOB-only table +/// the blob bunch itself carries the row-tracking fields, so the fallback reader must keep +/// _ROW_ID correct and report each row's _SEQUENCE_NUMBER from the layer that resolved it, +/// without ever exposing the internal placeholder sentinel. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { + arrow::FieldVector fields = {BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // old layer: rows 0-9 all real + std::vector> base_rows; + for (int32_t i = 0; i < 10; i++) { + base_rows.emplace_back("b" + std::to_string(i)); + } + auto base_array = MakeBlobUpdateArray(fields[0], base_rows); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // new layer: first_row_id=2, row_count=2, covering only rows 2-3 + auto update_array = MakeBlobUpdateArray(fields[0], {"u2", "u3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 10); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + int64_t old_layer_seq = seq_col->Value(0); + int64_t new_layer_seq = seq_col->Value(2); + ASSERT_GT(new_layer_seq, old_layer_seq); + for (int64_t i = 0; i < 10; i++) { + ASSERT_FALSE(blob_col->IsNull(i)); + std::string expected_blob = + (i == 2 || i == 3) ? "u" + std::to_string(i) : "b" + std::to_string(i); + // the leading and trailing gaps never expose the internal placeholder sentinel + ASSERT_EQ(blob_col->GetString(i), expected_blob) << "row " << i; + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_EQ(seq_col->Value(i), (i == 2 || i == 3) ? new_layer_seq : old_layer_seq) + << "row " << i; + } +} + +/// A row that is a placeholder in every BLOB layer degrades to a null blob but keeps its +/// _ROW_ID and reports -1 as its _SEQUENCE_NUMBER. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { + arrow::FieldVector fields = {BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // the base layer itself holds a placeholder at row 1, so after the second layer also + // leaves it untouched, row 1 is a placeholder in every layer + auto base_array = MakeBlobUpdateArray(fields[0], {"blob_a", "PH", "blob_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array = MakeBlobUpdateArray(fields[0], {"PH", "PH", "update_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 3); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + + ASSERT_EQ(blob_col->GetString(0), "blob_a"); + ASSERT_EQ(blob_col->GetString(2), "update_c"); + // the all-placeholder row: null blob, row id kept, sequence number -1 + ASSERT_TRUE(blob_col->IsNull(1)); + ASSERT_EQ(row_id_col->Value(1), 1); + ASSERT_EQ(seq_col->Value(1), -1); + for (int64_t i : {static_cast(0), static_cast(2)}) { + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_GE(seq_col->Value(i), 0); + } + ASSERT_GT(seq_col->Value(2), seq_col->Value(0)); +} + +/// A normal (full-row) write never interprets blob bytes: a user value whose bytes exactly +/// equal the placeholder sentinel must be stored verbatim (not persisted as a bin_length -2 +/// entry) and survive a later placeholder fallback unchanged instead of being resolved away. +TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // full-row write: row 0's blob is exactly the sentinel bytes + auto struct_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(), std::make_shared(), + std::make_shared()}); + auto f0_builder = static_cast(struct_builder.field_builder(0)); + auto f1_builder = static_cast(struct_builder.field_builder(1)); + auto b0_builder = static_cast(struct_builder.field_builder(2)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(1).ok()); + ASSERT_TRUE(f1_builder->Append("a").ok()); + ASSERT_TRUE(b0_builder->Append(sentinel_bytes.data(), sentinel_bytes.size()).ok()); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(2).ok()); + ASSERT_TRUE(f1_builder->Append("b").ok()); + ASSERT_TRUE(b0_builder->Append("normal", 6).ok()); + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_struct = std::dynamic_pointer_cast(src_array); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_struct})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // read back unchanged: the sentinel-equal bytes are a normal value + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_struct)); + + // a partial update leaves row 0 untouched: the fallback must resolve it back to the + // sentinel-equal user bytes rather than treating the stored value as a placeholder + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_result = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_result); + ASSERT_EQ(struct_result->length(), 2); + auto blob_col = + std::dynamic_pointer_cast(struct_result->GetFieldByName("b0")); + ASSERT_TRUE(blob_col); + ASSERT_FALSE(blob_col->IsNull(0)); + ASSERT_EQ(blob_col->GetString(0), sentinel_bytes); + ASSERT_EQ(blob_col->GetString(1), "updated_b"); +} + TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};