diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 409c58a50..09b813c36 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -130,6 +130,7 @@ set(PAIMON_COMMON_SRCS common/reader/predicate_batch_reader.cpp common/reader/prefetch_file_batch_reader_impl.cpp common/reader/reader_utils.cpp + common/reader/blob_fallback_batch_reader.cpp common/reader/blob_view_resolving_batch_reader.cpp common/reader/complete_row_kind_batch_reader.cpp common/reader/data_evolution_file_reader.cpp @@ -542,6 +543,7 @@ if(PAIMON_BUILD_TESTS) common/reader/prefetch_file_batch_reader_impl_test.cpp common/reader/reader_utils_test.cpp common/reader/complete_row_kind_batch_reader_test.cpp + common/reader/blob_fallback_batch_reader_test.cpp common/reader/blob_view_resolving_batch_reader_test.cpp common/reader/data_evolution_file_reader_test.cpp common/reader/data_evolution_array_test.cpp diff --git a/src/paimon/common/data/blob_defs.h b/src/paimon/common/data/blob_defs.h index b8bb6f308..aac13afe2 100644 --- a/src/paimon/common/data/blob_defs.h +++ b/src/paimon/common/data/blob_defs.h @@ -17,6 +17,8 @@ #pragma once #include +#include +#include namespace paimon { @@ -45,6 +47,60 @@ class BlobDefs { /// A bin_length value of -1 in the index indicates a null blob entry. static constexpr int64_t kNullBinLength = -1; + /// A bin_length value of -2 in the index indicates a placeholder blob entry, written by + /// data-evolution partial updates for rows whose blob value is not updated. A placeholder + /// entry occupies no file space; readers must fall back to an older blob file covering the + /// same row to resolve the value. Aligned with Java's BlobFormatWriter.PLACE_HOLDER_LENGTH. + static constexpr int64_t kPlaceholderBinLength = -2; + /// Sentinel bytes standing for a placeholder blob value in two internal channels: + /// + /// - Write channel: a data-evolution partial update (a blob-only column write, see + /// kWritePlaceholderKey) marks a not-updated row with these bytes, and the blob format + /// writer persists it as a bin_length -2 entry. Use PlaceholderSentinelView() to build + /// such write arrays. Outside that mode the writer never interprets values, so arbitrary + /// user bytes can never be turned into a placeholder entry. + /// - Read channel: a placeholder-aware reader (see kEmitPlaceholderSentinelKey) emits these + /// bytes for -2 entries so the fallback merge can identify placeholders after the batch + /// has passed through schema-mapping readers. + /// + /// Both channels escape legitimate values so user bytes can never collide with the + /// sentinel: a real value whose bytes start with the sentinel travels with one extra copy + /// of the sentinel prepended, and the consuming end strips it (an exact match is a + /// placeholder, a longer match is unescaped). Sentinel bytes are never stored in blob files + /// and never returned to users. Layout: version(1) + magic "BLOBPLHD"(8). + static constexpr char kPlaceholderSentinel[] = {0x01, 'B', 'L', 'O', 'B', 'P', 'L', 'H', 'D'}; + static constexpr int32_t kPlaceholderSentinelLength = 9; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// reader emits kPlaceholderSentinel for placeholder entries (escaping real values as + /// described above) instead of failing on them. Only the data-evolution blob fallback read + /// path sets this. + static constexpr char kEmitPlaceholderSentinelKey[] = "blob.internal.emit-placeholder-sentinel"; + /// Internal (non user-facing) format option, "false" by default: when "true", the blob + /// format writer applies the placeholder protocol to incoming values (an exact + /// kPlaceholderSentinel match becomes a bin_length -2 entry, a longer match is unescaped). + /// Only set for data-evolution partial updates, i.e. blob-only column writes of a table + /// with data evolution enabled; all other writes store bytes verbatim. + static constexpr char kWritePlaceholderKey[] = "blob.internal.write-placeholder"; + + /// The sentinel bytes for building a data-evolution partial-update write array: a row equal + /// to this view is persisted as a placeholder entry (see kWritePlaceholderKey). + static std::string_view PlaceholderSentinelView() { + return {kPlaceholderSentinel, static_cast(kPlaceholderSentinelLength)}; + } + + static bool IsPlaceholderSentinel(const char* data, size_t size) { + return size == static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } + + /// True when the bytes start with the placeholder sentinel (an exact match included). A + /// longer match is an escaped real value: strip the leading sentinel to recover it. A + /// producer for either placeholder channel must prepend one sentinel copy to any real value + /// this predicate matches. + static bool HasPlaceholderSentinelPrefix(const char* data, size_t size) { + return size >= static_cast(kPlaceholderSentinelLength) && + memcmp(data, kPlaceholderSentinel, kPlaceholderSentinelLength) == 0; + } /// Blob file format version. static constexpr int8_t kFileVersion = 1; /// Magic number identifying the start of each blob bin. diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.cpp b/src/paimon/common/reader/blob_fallback_batch_reader.cpp new file mode 100644 index 000000000..ae2547ffd --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.cpp @@ -0,0 +1,440 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/metrics/metrics_impl.h" +#include "paimon/common/reader/reader_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/arrow/mem_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" + +namespace paimon { + +Result> BlobFallbackBatchReader::Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool) { + if (sequence_groups.size() < 2) { + return Status::Invalid( + "Blob fallback needs at least two sequence groups; a single group should be read " + "sequentially."); + } + if (read_schema == nullptr) { + return Status::Invalid("Blob fallback read schema cannot be nullptr."); + } + if (read_batch_size <= 0) { + return Status::Invalid(fmt::format( + "Blob fallback read batch size '{}' should be larger than zero", read_batch_size)); + } + int32_t blob_field_idx = -1; + for (int32_t i = 0; i < read_schema->num_fields(); i++) { + if (BlobUtils::IsBlobField(read_schema->field(i))) { + if (blob_field_idx != -1) { + return Status::Invalid( + "Blob fallback read schema should contain exactly one blob field."); + } + blob_field_idx = i; + } + } + if (blob_field_idx == -1) { + return Status::Invalid("Blob fallback read schema should contain a blob field."); + } + int32_t row_id_field_idx = read_schema->GetFieldIndex(SpecialFields::RowId().Name()); + int32_t seq_num_field_idx = read_schema->GetFieldIndex(SpecialFields::SequenceNumber().Name()); + std::vector groups; + groups.reserve(sequence_groups.size()); + for (auto& segments : sequence_groups) { + if (segments.empty()) { + return Status::Invalid("Blob fallback sequence group should not be empty."); + } + for (const auto& segment : segments) { + if (segment.reader == nullptr && segment.gap_selected_ranges.empty()) { + return Status::Invalid( + "Blob fallback gap segment should cover at least one selected row id."); + } + } + GroupCursor cursor; + cursor.segments = std::move(segments); + groups.push_back(std::move(cursor)); + } + return std::unique_ptr( + new BlobFallbackBatchReader(std::move(groups), read_schema, blob_field_idx, + row_id_field_idx, seq_num_field_idx, read_batch_size, pool)); +} + +BlobFallbackBatchReader::BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool) + : groups_(std::move(groups)), + read_schema_(read_schema), + blob_field_idx_(blob_field_idx), + row_id_field_idx_(row_id_field_idx), + seq_num_field_idx_(seq_num_field_idx), + read_batch_size_(read_batch_size), + arrow_pool_(GetArrowPool(pool)) {} + +Result BlobFallbackBatchReader::FillWindow(size_t group_idx, int64_t want, + std::vector* chunks) { + GroupCursor& cursor = groups_[group_idx]; + int64_t collected = 0; + while (collected < want) { + if (!cursor.pending.empty()) { + const std::shared_ptr& front = cursor.pending.front(); + int64_t available = front->length() - cursor.pending_pos; + int64_t take = std::min(available, want - collected); + chunks->push_back(Chunk{front, cursor.pending_pos, take, {}, false}); + cursor.pending_pos += take; + collected += take; + if (cursor.pending_pos == front->length()) { + cursor.pending.pop_front(); + cursor.pending_pos = 0; + } + continue; + } + if (cursor.segment_idx >= cursor.segments.size()) { + // group exhausted; only the first group may define a shorter window + break; + } + Segment& segment = cursor.segments[cursor.segment_idx]; + if (segment.reader == nullptr) { + // gap segment: all rows are placeholders, stepped range by range so the row ids + // stay available for all-placeholder rows + if (cursor.gap_range_idx >= segment.gap_selected_ranges.size()) { + cursor.segment_idx++; + cursor.gap_range_idx = 0; + cursor.gap_range_pos = 0; + continue; + } + const Range& range = segment.gap_selected_ranges[cursor.gap_range_idx]; + int64_t remaining = range.Count() - cursor.gap_range_pos; + int64_t take = std::min(remaining, want - collected); + Chunk chunk{nullptr, 0, take, {}, false}; + if (row_id_field_idx_ >= 0) { + chunk.gap_row_ids.reserve(take); + for (int64_t k = 0; k < take; k++) { + chunk.gap_row_ids.push_back(range.from + cursor.gap_range_pos + k); + } + } + chunks->push_back(std::move(chunk)); + cursor.gap_range_pos += take; + collected += take; + if (cursor.gap_range_pos == range.Count()) { + cursor.gap_range_idx++; + cursor.gap_range_pos = 0; + } + continue; + } + PAIMON_ASSIGN_OR_RAISE(ReadBatchWithBitmap batch_with_bitmap, + segment.reader->NextBatchWithBitmap()); + if (BatchReader::IsEofBatch(batch_with_bitmap)) { + cursor.segment_idx++; + continue; + } + auto& [read_batch, bitmap] = batch_with_bitmap; + auto& [c_array, c_schema] = read_batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr src_array, + arrow::ImportArray(c_array.get(), c_schema.get())); + PAIMON_ASSIGN_OR_RAISE(arrow::ArrayVector selected_array_vec, + ReaderUtils::GenerateFilteredArrayVector(src_array, bitmap)); + for (const auto& selected_array : selected_array_vec) { + if (selected_array->length() == 0) { + continue; + } + auto struct_array = std::dynamic_pointer_cast(selected_array); + if (struct_array == nullptr) { + return Status::Invalid("Blob fallback expects file readers to emit struct arrays."); + } + cursor.pending.push_back(std::move(struct_array)); + } + } + return collected; +} + +Result> BlobFallbackBatchReader::ComputePlaceholderFlags( + std::vector* chunks, int64_t row_count) const { + std::vector flags(row_count, false); + int64_t pos = 0; + for (auto& chunk : *chunks) { + if (chunk.array == nullptr) { + // gap rows stand for placeholders + std::fill(flags.begin() + pos, flags.begin() + pos + chunk.length, true); + } else { + std::shared_ptr blob_col = chunk.array->field(blob_field_idx_); + auto binary_col = std::dynamic_pointer_cast(blob_col); + if (binary_col == nullptr) { + return Status::Invalid(fmt::format( + "Blob fallback expects the blob column to be large binary, but got {}", + blob_col->type()->ToString())); + } + for (int64_t k = 0; k < chunk.length; k++) { + int64_t idx = chunk.offset + k; + if (binary_col->IsNull(idx)) { + continue; + } + std::string_view value = binary_col->GetView(idx); + if (!BlobDefs::HasPlaceholderSentinelPrefix(value.data(), value.size())) { + continue; + } + if (BlobDefs::IsPlaceholderSentinel(value.data(), value.size())) { + flags[pos + k] = true; + } else { + // a real value escaped by the placeholder-aware blob reader + chunk.has_escaped = true; + } + } + } + pos += chunk.length; + } + return flags; +} + +Result> BlobFallbackBatchReader::UnescapeBlobSlice( + const std::shared_ptr& blob_col, int64_t offset, + int64_t length) const { + arrow::LargeBinaryBuilder builder(arrow_pool_.get()); + for (int64_t k = 0; k < length; k++) { + int64_t idx = offset + k; + if (blob_col->IsNull(idx)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.AppendNull()); + continue; + } + std::string_view value = blob_col->GetView(idx); + if (BlobDefs::HasPlaceholderSentinelPrefix(value.data(), value.size()) && + value.size() > static_cast(BlobDefs::kPlaceholderSentinelLength)) { + value.remove_prefix(BlobDefs::kPlaceholderSentinelLength); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(value.data(), value.size())); + } + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&array)); + return array; +} + +Result> BlobFallbackBatchReader::AssembleRowIdRun( + const std::vector& chunks, int64_t run_start, int64_t run_end) const { + arrow::ArrayVector pieces; + int64_t pos = 0; + for (const auto& chunk : chunks) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array != nullptr) { + std::shared_ptr column = chunk.array->field(row_id_field_idx_); + pieces.push_back(column->Slice(chunk.offset + (overlap_start - pos), + overlap_end - overlap_start)); + } else { + arrow::Int64Builder builder(arrow_pool_.get()); + for (int64_t r = overlap_start; r < overlap_end; r++) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Append(chunk.gap_row_ids[r - pos])); + } + std::shared_ptr piece; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder.Finish(&piece)); + pieces.push_back(std::move(piece)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + if (pieces.size() == 1) { + return pieces[0]; + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result> BlobFallbackBatchReader::AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const { + const auto row_count = static_cast(group_choice.size()); + arrow::ArrayVector pieces; + int64_t run_start = 0; + while (run_start < row_count) { + const int32_t group = group_choice[run_start]; + int64_t run_end = run_start + 1; + while (run_end < row_count && group_choice[run_end] == group) { + run_end++; + } + if (group < 0) { + // placeholder in every layer: the blob degrades to null; the row keeps its row id + // (taken from the newest group, which steps in lockstep), reports -1 as its + // sequence number, and returns null for every other field + if (field_idx == row_id_field_idx_) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr row_id_piece, + AssembleRowIdRun(group_chunks[0], run_start, run_end)); + pieces.push_back(std::move(row_id_piece)); + } else if (field_idx == seq_num_field_idx_) { + arrow::Int64Scalar seq_scalar(-1); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr seq_piece, + arrow::MakeArrayFromScalar(seq_scalar, run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(seq_piece)); + } else { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW( + std::shared_ptr null_piece, + arrow::MakeArrayOfNull(read_schema_->field(field_idx)->type(), + run_end - run_start, arrow_pool_.get())); + pieces.push_back(std::move(null_piece)); + } + } else { + int64_t pos = 0; + for (const auto& chunk : group_chunks[group]) { + int64_t overlap_start = std::max(run_start, pos); + int64_t overlap_end = std::min(run_end, pos + chunk.length); + if (overlap_start < overlap_end) { + if (chunk.array == nullptr) { + return Status::Invalid( + "Unexpected: a gap row was chosen as a blob fallback result."); + } + std::shared_ptr column = chunk.array->field(field_idx); + int64_t slice_offset = chunk.offset + (overlap_start - pos); + int64_t slice_length = overlap_end - overlap_start; + if (field_idx == blob_field_idx_ && chunk.has_escaped) { + auto binary_col = + std::dynamic_pointer_cast(column); + if (binary_col == nullptr) { + return Status::Invalid( + "Blob fallback expects the blob column to be large binary."); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr unescaped, + UnescapeBlobSlice(binary_col, slice_offset, slice_length)); + pieces.push_back(std::move(unescaped)); + } else { + pieces.push_back(column->Slice(slice_offset, slice_length)); + } + } + pos += chunk.length; + if (pos >= run_end) { + break; + } + } + } + run_start = run_end; + } + if (pieces.size() == 1 && pieces[0]->offset() == 0) { + return pieces[0]; + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr concat_array, + arrow::Concatenate(pieces, arrow_pool_.get())); + return concat_array; +} + +Result BlobFallbackBatchReader::NextBatch() { + if (closed_) { + return Status::Invalid("blob fallback batch reader is closed"); + } + std::vector> group_chunks(groups_.size()); + // the first (newest) group defines the window; the others must step in lockstep + PAIMON_ASSIGN_OR_RAISE(int64_t row_count, FillWindow(0, read_batch_size_, &group_chunks[0])); + for (size_t g = 1; g < groups_.size(); g++) { + int64_t expect = row_count == 0 ? 1 : row_count; + PAIMON_ASSIGN_OR_RAISE(int64_t got, FillWindow(g, expect, &group_chunks[g])); + int64_t expected = row_count == 0 ? 0 : row_count; + if (got != expected) { + return Status::Invalid(fmt::format( + "All sequence groups of a blob fallback read should have the same number of " + "rows: group {} yielded {} rows in a window of {}", + g, got, expected)); + } + } + if (row_count == 0) { + return BatchReader::MakeEofBatch(); + } + + std::vector> placeholder_flags(groups_.size()); + for (size_t g = 0; g < groups_.size(); g++) { + PAIMON_ASSIGN_OR_RAISE(placeholder_flags[g], + ComputePlaceholderFlags(&group_chunks[g], row_count)); + } + // per row, the first group in max-sequence order with a real entry wins; -1 means the row is + // a placeholder in every group + std::vector group_choice(row_count, -1); + for (int64_t r = 0; r < row_count; r++) { + for (size_t g = 0; g < groups_.size(); g++) { + if (!placeholder_flags[g][r]) { + group_choice[r] = static_cast(g); + break; + } + } + } + + arrow::ArrayVector columns; + columns.reserve(read_schema_->num_fields()); + for (int32_t field_idx = 0; field_idx < read_schema_->num_fields(); field_idx++) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr column, + AssembleColumn(field_idx, group_choice, group_chunks)); + columns.push_back(std::move(column)); + } + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr target_array, + arrow::StructArray::Make(columns, read_schema_->fields())); + std::unique_ptr c_array = std::make_unique(); + std::unique_ptr c_schema = std::make_unique(); + PAIMON_RETURN_NOT_OK_FROM_ARROW( + arrow::ExportArray(*target_array, c_array.get(), c_schema.get())); + return std::make_pair(std::move(c_array), std::move(c_schema)); +} + +Result BlobFallbackBatchReader::NextBatchWithBitmap() { + PAIMON_ASSIGN_OR_RAISE(ReadBatch batch, NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + return BatchReader::MakeEofBatchWithBitmap(); + } + return ReaderUtils::AddAllValidBitmap(std::move(batch)); +} + +void BlobFallbackBatchReader::Close() { + for (auto& group : groups_) { + group.pending.clear(); + for (auto& segment : group.segments) { + if (segment.reader) { + segment.reader->Close(); + } + } + } + closed_ = true; +} + +std::shared_ptr BlobFallbackBatchReader::GetReaderMetrics() const { + auto metrics = std::make_shared(); + for (const auto& group : groups_) { + for (const auto& segment : group.segments) { + if (segment.reader) { + auto reader_metrics = segment.reader->GetReaderMetrics(); + if (reader_metrics) { + metrics->Merge(reader_metrics); + } + } + } + } + return metrics; +} + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader.h b/src/paimon/common/reader/blob_fallback_batch_reader.h new file mode 100644 index 000000000..040cf316c --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader.h @@ -0,0 +1,174 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "arrow/type.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/reader/file_batch_reader.h" +#include "paimon/result.h" +#include "paimon/status.h" +#include "paimon/utils/range.h" + +namespace arrow { +class Array; +class StructArray; +class LargeBinaryArray; +} // namespace arrow + +namespace paimon { + +/// Merges the blob files of one data-evolution blob bunch that span multiple max sequence +/// number layers, resolving placeholder entries row by row. Aligned with Java's +/// BlobFallbackRecordReader / AllPlaceholdersRecordReader. +/// +/// A data-evolution partial update rewrites only the touched rows of a blob column; the new blob +/// file records every untouched row as a placeholder entry. Reading therefore needs, per row, the +/// value from the newest layer that holds a real (non-placeholder) entry: +/// +/// 1. The caller groups the blob files by max sequence number (one group per layer, newest +/// first) and, inside each group, orders them by first row id. Row id ranges the group's +/// files do not cover are represented by gap segments, which stand for all-placeholder rows. +/// 2. All groups span the same overall row id range, so with the same row-ranges selection +/// applied they yield the same number of rows and can be stepped in lockstep. +/// 3. Each output row takes the first group, in max-sequence order, whose row is not a +/// placeholder. Placeholder rows are identified by the BlobDefs::kPlaceholderSentinel bytes, +/// emitted by the blob format reader when BlobDefs::kEmitPlaceholderSentinelKey is set; a +/// real value starting with those bytes arrives escaped by one extra sentinel copy, which +/// this reader strips (see BlobDefs::HasPlaceholderSentinelPrefix). +/// 4. A row that is a placeholder in every group degrades to a null blob: it keeps its +/// _ROW_ID, reports -1 as its _SEQUENCE_NUMBER, and returns null for every other field. +class BlobFallbackBatchReader : public BatchReader { + public: + /// One piece of a sequence group: either a reader over a single blob file (already wrapped + /// with the usual per-file mapping readers and row selection), or a virtual gap standing for + /// row ids the group's files do not cover. Segments must be ordered by ascending row id. + struct Segment { + /// File segment: emits the rows of one blob file. Null for a gap segment. + std::unique_ptr reader; + /// Gap segment only: the selected row ids the gap emits (all placeholders), as sorted + /// disjoint ranges. Must not be empty for a gap segment. + std::vector gap_selected_ranges; + }; + + /// `sequence_groups` must be ordered by descending max sequence number and contain at least + /// two groups (a single group needs no fallback). `read_schema` is the schema every file + /// reader emits; it must contain exactly one blob field and may additionally contain the + /// row-tracking fields _ROW_ID and _SEQUENCE_NUMBER (completed per file by + /// CompleteRowTrackingFieldsBatchReader), which stay correct for rows that are a + /// placeholder in every layer. + static Result> Create( + std::vector>&& sequence_groups, + const std::shared_ptr& read_schema, int32_t read_batch_size, + const std::shared_ptr& pool); + + Result NextBatch() override; + + Result NextBatchWithBitmap() override; + + void Close() override; + + std::shared_ptr GetReaderMetrics() const override; + + private: + /// A run of consecutive rows already fetched from one group: `array` rows + /// [offset, offset + length) for a file segment, or `length` placeholder rows for a gap + /// segment (array is null). + struct Chunk { + std::shared_ptr array; + int64_t offset = 0; + int64_t length = 0; + /// Gap chunk only: the row id of each of the `length` rows, filled when the read + /// schema contains _ROW_ID so all-placeholder rows can keep their row id. + std::vector gap_row_ids; + /// File chunk only: true when the blob column holds at least one escaped value + /// (BlobDefs::HasPlaceholderSentinelPrefix but longer than the sentinel), which must be + /// unescaped before it reaches the output. + bool has_escaped = false; + }; + + /// Read progress of one sequence group. Movable only: deleting copy makes vector growth + /// move the cursors instead of instantiating Segment's deleted copy constructor. + struct GroupCursor { + GroupCursor() = default; + GroupCursor(const GroupCursor&) = delete; + GroupCursor& operator=(const GroupCursor&) = delete; + GroupCursor(GroupCursor&&) = default; + GroupCursor& operator=(GroupCursor&&) = default; + + std::vector segments; + size_t segment_idx = 0; + /// Position inside the current gap segment: index into gap_selected_ranges and the + /// number of rows already emitted from that range. + size_t gap_range_idx = 0; + int64_t gap_range_pos = 0; + /// Rows fetched from the current file segment but not yet consumed. + std::deque> pending; + int64_t pending_pos = 0; + }; + + BlobFallbackBatchReader(std::vector&& groups, + const std::shared_ptr& read_schema, + int32_t blob_field_idx, int32_t row_id_field_idx, + int32_t seq_num_field_idx, int32_t read_batch_size, + const std::shared_ptr& pool); + + /// Collects up to `want` rows from the group into chunks. Only the first group may come up + /// short (which defines the window size); any later group ending early is a misalignment. + Result FillWindow(size_t group_idx, int64_t want, std::vector* chunks); + + /// Flags each of the `row_count` window rows of the given chunks as placeholder or not, + /// and marks chunks holding escaped blob values (see Chunk::has_escaped). + Result> ComputePlaceholderFlags(std::vector* chunks, + int64_t row_count) const; + + /// Assembles one output column by stitching, per run of rows choosing the same group, + /// slices of that group's chunks (unescaping escaped blob values). For rows choosing no + /// group (placeholder in every layer), _ROW_ID is kept, _SEQUENCE_NUMBER becomes -1, and + /// every other field becomes null. + Result> AssembleColumn( + int32_t field_idx, const std::vector& group_choice, + const std::vector>& group_chunks) const; + + /// Unescapes the blob column rows [offset, offset + length) of a file chunk: values + /// starting with the sentinel bytes lose the leading sentinel copy. + Result> UnescapeBlobSlice( + const std::shared_ptr& blob_col, int64_t offset, + int64_t length) const; + + /// Assembles the _ROW_ID values of rows [run_start, run_end) from the given group's chunks; + /// gap chunks contribute their synthesized row ids. + Result> AssembleRowIdRun(const std::vector& chunks, + int64_t run_start, + int64_t run_end) const; + + std::vector groups_; + std::shared_ptr read_schema_; + const int32_t blob_field_idx_; + /// Index of _ROW_ID / _SEQUENCE_NUMBER in the read schema, -1 when not read. + const int32_t row_id_field_idx_; + const int32_t seq_num_field_idx_; + const int32_t read_batch_size_; + std::shared_ptr arrow_pool_; + bool closed_ = false; +}; + +} // namespace paimon diff --git a/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp new file mode 100644 index 000000000..6dbe761aa --- /dev/null +++ b/src/paimon/common/reader/blob_fallback_batch_reader_test.cpp @@ -0,0 +1,362 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/reader/blob_fallback_batch_reader.h" + +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/util/range.h" +#include "gtest/gtest.h" +#include "paimon/common/data/blob_defs.h" +#include "paimon/common/data/blob_utils.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/mock/mock_file_batch_reader.h" +#include "paimon/testing/utils/read_result_collector.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +/// "PH" stands for a placeholder row (the sentinel bytes emitted by the placeholder-aware blob +/// reader), std::nullopt for null. +using BlobRows = std::vector>; + +class BlobFallbackBatchReaderTest : public ::testing::Test { + public: + void SetUp() override { + pool_ = GetDefaultPool(); + struct_type_ = arrow::struct_({BlobUtils::ToArrowField("blob_col", true)}); + read_schema_ = arrow::schema(struct_type_->fields()); + } + + static std::string Sentinel() { + return std::string(BlobDefs::PlaceholderSentinelView()); + } + + std::shared_ptr MakeBlobStruct(const BlobRows& rows) const { + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + } + + /// One segment of a group: File(rows) for a file segment, Gap(n) for a placeholder gap + /// of n selected rows (with synthetic row ids when the schema does not read them, or the + /// given ranges via GapRanges). + struct SegmentSpec { + std::vector gap_ranges; + std::optional file_rows; + static SegmentSpec Gap(int64_t rows) { + return SegmentSpec{{Range(0, rows - 1)}, std::nullopt}; + } + static SegmentSpec GapRanges(std::vector ranges) { + return SegmentSpec{std::move(ranges), std::nullopt}; + } + static SegmentSpec File(BlobRows rows) { + return SegmentSpec{{}, std::move(rows)}; + } + }; + + std::vector MakeGroup(const std::vector& specs, + int32_t file_batch_size) const { + std::vector segments; + for (const auto& spec : specs) { + if (spec.file_rows) { + auto reader = std::make_unique(MakeBlobStruct(*spec.file_rows), + struct_type_, file_batch_size); + segments.push_back(BlobFallbackBatchReader::Segment{std::move(reader), {}}); + } else { + segments.push_back(BlobFallbackBatchReader::Segment{nullptr, spec.gap_ranges}); + } + } + return segments; + } + + /// Runs the fallback over the groups with several batch sizes and compares to expected rows. + void CheckFallback(const std::vector>& group_specs, + const BlobRows& expected_rows) const { + auto expected_array = MakeBlobStruct(expected_rows); + for (auto batch_size : arrow::internal::Iota(1, 8)) { + for (auto file_batch_size : {1, 3, 1024}) { + std::vector> groups; + groups.reserve(group_specs.size()); + for (const auto& specs : group_specs) { + groups.push_back(MakeGroup(specs, file_batch_size)); + } + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + auto expected_chunk_array = std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + + protected: + std::shared_ptr pool_; + std::shared_ptr struct_type_; + std::shared_ptr read_schema_; +}; + +TEST_F(BlobFallbackBatchReaderTest, TestBasicFallback) { + // newer layer updates row 1 only; rows 0 and 2 fall back to the older layer + CheckFallback( + {{SegmentSpec::File({"PH", "u1", "PH"})}, {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "u1", "b2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestGapPadding) { + // the newer layer only covers rows 2-3; the gaps stand for placeholders + CheckFallback({{SegmentSpec::Gap(2), SegmentSpec::File({"u2", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "b1", "u2", "b3"}); + // trailing gap + CheckFallback({{SegmentSpec::File({"PH", "u1"}), SegmentSpec::Gap(2)}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"b0", "u1", "b2", "b3"}); + // middle gap between two files of one layer + CheckFallback({{SegmentSpec::File({"u0"}), SegmentSpec::Gap(2), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3"})}}, + {"u0", "b1", "b2", "u3"}); + // a gap segment covering multiple disjoint selected ranges + CheckFallback({{SegmentSpec::GapRanges({Range(0, 0), Range(2, 2)}), SegmentSpec::File({"u3"})}, + {SegmentSpec::File({"b0", "b2", "b3"})}}, + {"b0", "b2", "u3"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestAllPlaceholdersBecomesNull) { + // a row that is a placeholder in every layer degrades to null + CheckFallback({{SegmentSpec::File({"PH", "PH"})}, {SegmentSpec::File({"b0", "PH"})}}, + {"b0", std::nullopt}); + CheckFallback({{SegmentSpec::Gap(2)}, {SegmentSpec::File({"PH", "PH"})}}, + {std::nullopt, std::nullopt}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestNullIsNotPlaceholder) { + // a real null in a newer layer wins: null means "updated to null", not "not updated" + CheckFallback({{SegmentSpec::File({std::nullopt, "u1"})}, {SegmentSpec::File({"b0", "b1"})}}, + {std::nullopt, "u1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestEscapedSentinelIsNotPlaceholder) { + // a real value whose bytes equal the sentinel arrives escaped (sentinel + value) from the + // placeholder-aware blob reader; the merge must unescape it, not treat it as a placeholder + std::string escaped_sentinel = Sentinel() + Sentinel(); + CheckFallback( + {{SegmentSpec::File({"PH", "u1"})}, {SegmentSpec::File({escaped_sentinel, "b1"})}}, + {Sentinel(), "u1"}); + // an escaped sentinel-prefixed value unescapes to the original bytes, also when it wins as + // the newest layer + std::string escaped_prefixed = Sentinel() + Sentinel() + "suffix"; + CheckFallback( + {{SegmentSpec::File({escaped_prefixed, "PH"})}, {SegmentSpec::File({"b0", "b1"})}}, + {Sentinel() + "suffix", "b1"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestThreeLayers) { + CheckFallback({{SegmentSpec::File({"PH", "PH", "u2"})}, + {SegmentSpec::File({"PH", "m1", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2"})}}, + {"b0", "m1", "u2"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestLayeredFilesAndGaps) { + // mirrors the compacted-sequence-groups shape: layers partially cover [0, 9] + CheckFallback( + {{SegmentSpec::Gap(6), SegmentSpec::File({"u66", "PH"}), SegmentSpec::Gap(1), + SegmentSpec::File({"u69"})}, + {SegmentSpec::File({"u40", "PH", "PH", "PH"}), SegmentSpec::Gap(4), + SegmentSpec::File({"u48", "PH"})}, + {SegmentSpec::File({"b0", "b1", "b2", "b3", "b4", "b5", "b6", "b7", "b8", "b9"})}}, + {"u40", "b1", "b2", "b3", "b4", "b5", "u66", "b7", "u48", "u69"}); +} + +TEST_F(BlobFallbackBatchReaderTest, TestRowTrackingFieldsPreserved) { + // Row-tracking projections: resolved rows keep their layer's row id and sequence number; + // an all-placeholder row keeps its row id (here provided by the newest group's gap + // segment), reports -1 as its sequence number, and degrades the blob to null. Covers the + // schema variants {blob, _ROW_ID, _SEQUENCE_NUMBER}, {blob, _ROW_ID} and + // {blob, _SEQUENCE_NUMBER}. + struct RowSpec { + std::optional blob; + int64_t row_id; + int64_t seq_num; + }; + for (bool with_row_id : {true, false}) { + for (bool with_seq_num : {true, false}) { + if (!with_row_id && !with_seq_num) { + continue; + } + arrow::FieldVector fields = {BlobUtils::ToArrowField("blob_col", true)}; + if (with_row_id) { + fields.push_back(SpecialFields::RowId().field_); + } + if (with_seq_num) { + fields.push_back(SpecialFields::SequenceNumber().field_); + } + auto struct_type = arrow::struct_(fields); + auto schema = arrow::schema(fields); + + auto make_rows = [&](const std::vector& rows) { + std::vector> field_builders = { + std::make_shared()}; + for (size_t i = 1; i < fields.size(); i++) { + field_builders.push_back(std::make_shared()); + } + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + std::move(field_builders)); + auto blob_builder = + static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row.blob) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row.blob == "PH") { + std::string sentinel = Sentinel(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row.blob->data(), row.blob->size()).ok()); + } + int32_t next_field = 1; + if (with_row_id) { + auto builder = static_cast( + struct_builder.field_builder(next_field++)); + EXPECT_TRUE(builder->Append(row.row_id).ok()); + } + if (with_seq_num) { + auto builder = static_cast( + struct_builder.field_builder(next_field)); + EXPECT_TRUE(builder->Append(row.seq_num).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return array; + }; + + for (auto batch_size : arrow::internal::Iota(1, 5)) { + for (auto file_batch_size : {1, 1024}) { + // newest layer (seq 20) covers only row 2; rows 0-1 are a gap + std::vector newest; + newest.push_back(BlobFallbackBatchReader::Segment{nullptr, {Range(0, 1)}}); + newest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique(make_rows({{"u2", 2, 20}}), + struct_type, file_batch_size), + {}}); + // oldest layer (seq 10) covers rows 0-2, row 1 is a placeholder there too + std::vector oldest; + oldest.push_back(BlobFallbackBatchReader::Segment{ + std::make_unique( + make_rows({{"b0", 0, 10}, {"PH", 1, 10}, {"PH", 2, 10}}), struct_type, + file_batch_size), + {}}); + std::vector> groups; + groups.push_back(std::move(newest)); + groups.push_back(std::move(oldest)); + + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFallbackBatchReader::Create(std::move(groups), schema, + batch_size, pool_)); + ASSERT_OK_AND_ASSIGN( + auto result, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + reader->Close(); + + // row 0 falls back to seq 10, row 1 is all-placeholder (null blob, row id + // kept, seq -1), row 2 takes seq 20 + auto expected_array = + make_rows({{"b0", 0, 10}, {std::nullopt, 1, -1}, {"u2", 2, 20}}); + auto expected_chunk_array = + std::make_shared(expected_array); + ASSERT_TRUE(result->Equals(expected_chunk_array)) + << "with_row_id=" << with_row_id << " with_seq_num=" << with_seq_num + << " batch_size=" << batch_size << " file_batch_size=" << file_batch_size + << "\nresult: " << result->ToString() + << "\nexpected: " << expected_chunk_array->ToString(); + } + } + } + } +} + +TEST_F(BlobFallbackBatchReaderTest, TestMisalignedGroupsFail) { + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"PH", "u1", "PH"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b0", "b1"})}, 1024)); + ASSERT_OK_AND_ASSIGN( + auto reader, BlobFallbackBatchReader::Create(std::move(groups), read_schema_, 1024, pool_)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "same number of rows"); +} + +TEST_F(BlobFallbackBatchReaderTest, TestCreateValidation) { + // a single group needs no fallback + std::vector> single_group; + single_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(single_group), read_schema_, 1024, pool_), + "at least two sequence groups"); + + // the read schema must contain a blob field + std::vector> groups; + groups.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + groups.push_back(MakeGroup({SegmentSpec::File({"b1"})}, 1024)); + auto plain_schema = + arrow::schema({arrow::field("not_blob", arrow::large_binary(), /*nullable=*/true)}); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(groups), plain_schema, 1024, pool_), + "should contain a blob field"); + + // groups must not be empty + std::vector> with_empty_group; + with_empty_group.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_group.emplace_back(); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_group), read_schema_, 1024, pool_), + "should not be empty"); + + // a gap segment must cover at least one selected row id + std::vector> with_empty_gap; + with_empty_gap.push_back(MakeGroup({SegmentSpec::File({"b0"})}, 1024)); + with_empty_gap.push_back(MakeGroup({SegmentSpec::GapRanges({})}, 1024)); + ASSERT_NOK_WITH_MSG( + BlobFallbackBatchReader::Create(std::move(with_empty_gap), read_schema_, 1024, pool_), + "at least one selected row id"); +} + +} // namespace paimon::test diff --git a/src/paimon/core/append/append_only_writer.cpp b/src/paimon/core/append/append_only_writer.cpp index 21d2b8db7..41426a2bf 100644 --- a/src/paimon/core/append/append_only_writer.cpp +++ b/src/paimon/core/append/append_only_writer.cpp @@ -208,11 +208,11 @@ AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetDataFileWriterFactory( AppendOnlyWriter::WriterFactory AppendOnlyWriter::GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const { + const std::optional>& write_cols, bool write_placeholder) const { std::shared_ptr path_factory = path_factory_; return std::make_shared(options_, schema_id_, single_field_schema, write_cols, seq_num_counter_, path_factory, - memory_pool_); + write_placeholder, memory_pool_); } AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWriter( @@ -220,8 +220,13 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri // Multiple blob fields are supported. Each blob field gets its own rolling file writer // via MultipleBlobFileWriter. auto blob_schema = schemas.blob_schema; + // A data-evolution write touching only blob columns is a partial update: its rows may mark + // untouched blobs with the placeholder sentinel (see BlobDefs::kWritePlaceholderKey). Any + // write that also carries non-blob columns stores blob bytes verbatim. + bool write_placeholder = + options_.DataEvolutionEnabled() && schemas.main_schema->num_fields() == 0; MultipleBlobFileWriter::BlobWriterCreator blob_writer_creator = - [this, blob_schema](const std::string& blob_field_name) + [this, blob_schema, write_placeholder](const std::string& blob_field_name) -> Result< std::unique_ptr>>> { // Create a single-field schema for this blob field @@ -233,7 +238,7 @@ AppendOnlyWriter::RollingFileWriterResult AppendOnlyWriter::CreateRollingBlobWri auto single_field_schema = arrow::schema({field}); std::vector write_cols = {blob_field_name}; auto single_blob_file_writer_factory = - GetBlobFileWriterFactory(single_field_schema, write_cols); + GetBlobFileWriterFactory(single_field_schema, write_cols, write_placeholder); return std::make_unique>>( options_.GetBlobTargetFileSize(), single_blob_file_writer_factory); }; diff --git a/src/paimon/core/append/append_only_writer.h b/src/paimon/core/append/append_only_writer.h index 4e64c0d99..066bd0259 100644 --- a/src/paimon/core/append/append_only_writer.h +++ b/src/paimon/core/append/append_only_writer.h @@ -110,7 +110,7 @@ class AppendOnlyWriter : public BatchWriter { WriterFactory GetBlobFileWriterFactory( const std::shared_ptr& single_field_schema, - const std::optional>& write_cols) const; + const std::optional>& write_cols, bool write_placeholder) const; Status TrySyncLatestCompaction(bool blocking); Status UpdateCompactDeletionFile(const std::shared_ptr& new_deletion_file); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.cpp b/src/paimon/core/io/blob_data_file_writer_factory.cpp index aca03e7ab..86aa4a83f 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.cpp +++ b/src/paimon/core/io/blob_data_file_writer_factory.cpp @@ -17,8 +17,11 @@ #include "paimon/core/io/blob_data_file_writer_factory.h" #include +#include +#include #include +#include "paimon/common/data/blob_defs.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_path_factory.h" #include "paimon/core/manifest/file_source.h" @@ -33,20 +36,25 @@ BlobDataFileWriterFactory::BlobDataFileWriterFactory( const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, - const std::shared_ptr& path_factory, + const std::shared_ptr& path_factory, bool write_placeholder, const std::shared_ptr& pool) : DataFileWriterFactory(options, schema_id, pool), file_schema_(file_schema), write_cols_(write_cols), seq_num_counter_(seq_num_counter), - path_factory_(path_factory) {} + path_factory_(path_factory), + write_placeholder_(write_placeholder) {} Result>>> BlobDataFileWriterFactory::CreateWriter() const { std::shared_ptr seq_num_counter = options_.DataEvolutionEnabled() ? std::make_shared(0) : seq_num_counter_; + std::map format_options = options_.ToMap(); + if (write_placeholder_) { + format_options[BlobDefs::kWritePlaceholderKey] = "true"; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr format, - FileFormatFactory::Get("blob", options_.ToMap())); + FileFormatFactory::Get("blob", format_options)); PAIMON_ASSIGN_OR_RAISE(WriterResources resources, CreateWriterResources(*format, file_schema_, /*create_stats_extractor=*/true)); diff --git a/src/paimon/core/io/blob_data_file_writer_factory.h b/src/paimon/core/io/blob_data_file_writer_factory.h index e79d42e13..3ff813f97 100644 --- a/src/paimon/core/io/blob_data_file_writer_factory.h +++ b/src/paimon/core/io/blob_data_file_writer_factory.h @@ -43,12 +43,15 @@ class BlobDataFileWriterFactory : public DataFileWriterFactory, public SingleFileWriterFactory<::ArrowArray*, std::shared_ptr> { public: + /// `write_placeholder` marks a data-evolution partial-update write: the blob format + /// writer is created with BlobDefs::kWritePlaceholderKey and persists placeholder + /// sentinel rows as placeholder entries. BlobDataFileWriterFactory(const CoreOptions& options, int64_t schema_id, const std::shared_ptr& file_schema, const std::optional>& write_cols, const std::shared_ptr& seq_num_counter, const std::shared_ptr& path_factory, - const std::shared_ptr& pool); + bool write_placeholder, const std::shared_ptr& pool); Result>>> CreateWriter() const override; @@ -58,6 +61,7 @@ class BlobDataFileWriterFactory std::optional> write_cols_; std::shared_ptr seq_num_counter_; std::shared_ptr path_factory_; + bool write_placeholder_ = false; }; } // namespace paimon diff --git a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp index 29b610b16..0c724d88a 100644 --- a/src/paimon/core/io/complete_row_tracking_fields_reader.cpp +++ b/src/paimon/core/io/complete_row_tracking_fields_reader.cpp @@ -39,9 +39,18 @@ CompleteRowTrackingFieldsBatchReader::CompleteRowTrackingFieldsBatchReader( Status CompleteRowTrackingFieldsBatchReader::SetReadSchema( ::ArrowSchema* read_schema, const std::shared_ptr& predicate, const std::optional& selection_bitmap) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr<::ArrowSchema> c_file_schema, reader_->GetFileSchema()); - PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr file_schema, - arrow::ImportSchema(c_file_schema.get())); + // A format without a self-describing file schema (e.g. blob) cannot hold the special + // fields, so treat it as an empty file schema and synthesize them from the file meta. + std::shared_ptr file_schema; + Result> file_schema_result = reader_->GetFileSchema(); + if (file_schema_result.ok()) { + std::unique_ptr<::ArrowSchema> c_file_schema = std::move(file_schema_result).value(); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(file_schema, arrow::ImportSchema(c_file_schema.get())); + } else if (file_schema_result.status().IsNotImplemented()) { + file_schema = arrow::schema(arrow::FieldVector{}); + } else { + return file_schema_result.status(); + } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr arrow_schema, arrow::ImportSchema(read_schema)); read_schema_ = arrow_schema; diff --git a/src/paimon/core/mergetree/lookup_levels.cpp b/src/paimon/core/mergetree/lookup_levels.cpp index ab323a235..4cea81edf 100644 --- a/src/paimon/core/mergetree/lookup_levels.cpp +++ b/src/paimon/core/mergetree/lookup_levels.cpp @@ -345,7 +345,8 @@ Status LookupLevels::CreateSstFileFromDataFile(const std::shared_ptr> raw_readers, split_read_->CreateRawFileReaders(partition_, {file}, read_schema_, /*predicate=*/nullptr, dv_factory_, - /*row_ranges=*/std::nullopt, data_file_path_factory_)); + /*row_ranges=*/std::nullopt, data_file_path_factory_, + /*extra_format_options=*/{})); if (raw_readers.size() != 1) { return Status::Invalid("Unexpected, CreateSstFileFromDataFile only create single reader"); } diff --git a/src/paimon/core/operation/abstract_split_read.cpp b/src/paimon/core/operation/abstract_split_read.cpp index dc31aff23..9aa4ec921 100644 --- a/src/paimon/core/operation/abstract_split_read.cpp +++ b/src/paimon/core/operation/abstract_split_read.cpp @@ -75,7 +75,8 @@ Result>> AbstractSplitRead::CreateR const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const { + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const { if (data_files.empty()) { return std::vector>(); } @@ -89,7 +90,7 @@ Result>> AbstractSplitRead::CreateR auto data_file_path = data_file_path_factory->ToPath(file); PAIMON_ASSIGN_OR_RAISE(std::string data_file_identifier, file->FileFormat()); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, - PrepareReaderBuilder(data_file_identifier)); + PrepareReaderBuilder(data_file_identifier, extra_format_options)); PAIMON_ASSIGN_OR_RAISE( std::unique_ptr file_reader, CreateFieldMappingReader(data_file_path, file, partition, reader_builder.get(), @@ -120,9 +121,14 @@ Result> AbstractSplitRead::ApplyPredicateFilterIfNe } Result> AbstractSplitRead::PrepareReaderBuilder( - const std::string& format_identifier) const { + const std::string& format_identifier, + const std::map& extra_format_options) const { + std::map format_options = options_.ToMap(); + for (const auto& [key, value] : extra_format_options) { + format_options[key] = value; + } PAIMON_ASSIGN_OR_RAISE(std::unique_ptr file_format, - FileFormatFactory::Get(format_identifier, options_.ToMap())); + FileFormatFactory::Get(format_identifier, format_options)); PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader_builder, file_format->CreateReaderBuilder(options_.GetReadBatchSize())); reader_builder->WithMemoryPool(pool_); diff --git a/src/paimon/core/operation/abstract_split_read.h b/src/paimon/core/operation/abstract_split_read.h index 89c94fbf4..933c284d5 100644 --- a/src/paimon/core/operation/abstract_split_read.h +++ b/src/paimon/core/operation/abstract_split_read.h @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -60,12 +61,16 @@ class AbstractSplitRead : public SplitRead { public: ~AbstractSplitRead() override = default; + /// `extra_format_options` are merged over the table options when building the format + /// reader, e.g. to switch the blob format reader into placeholder-aware mode for the + /// data-evolution blob fallback read path. Result>> CreateRawFileReaders( const BinaryRow& partition, const std::vector>& data_files, const std::shared_ptr& read_schema, const std::shared_ptr& predicate, DeletionVector::Factory dv_factory, const std::optional>& row_ranges, - const std::shared_ptr& data_file_path_factory) const; + const std::shared_ptr& data_file_path_factory, + const std::map& extra_format_options) const; protected: AbstractSplitRead(const std::shared_ptr& path_factory, @@ -95,7 +100,8 @@ class AbstractSplitRead : public SplitRead { private: Result> PrepareReaderBuilder( - const std::string& format_identifier) const; + const std::string& format_identifier, + const std::map& extra_format_options) const; Result> CreateFileBatchReader( const std::string& file_format_identifier, const std::string& data_file_path, diff --git a/src/paimon/core/operation/data_evolution_split_read.cpp b/src/paimon/core/operation/data_evolution_split_read.cpp index 1aecaa6d8..28017ccfc 100644 --- a/src/paimon/core/operation/data_evolution_split_read.cpp +++ b/src/paimon/core/operation/data_evolution_split_read.cpp @@ -16,6 +16,8 @@ #include "paimon/core/operation/data_evolution_split_read.h" +#include +#include #include #include #include @@ -28,10 +30,12 @@ #include "arrow/array/array_nested.h" #include "arrow/c/bridge.h" #include "paimon/common/catalog/catalog_context.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" #include "paimon/common/file_index/bitmap/apply_bitmap_index_batch_reader.h" #include "paimon/common/global_index/complete_index_score_batch_reader.h" +#include "paimon/common/reader/blob_fallback_batch_reader.h" #include "paimon/common/reader/blob_view_resolving_batch_reader.h" #include "paimon/common/reader/complete_row_kind_batch_reader.h" #include "paimon/common/reader/concat_batch_reader.h" @@ -45,73 +49,63 @@ #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/utils/blob_view_lookup.h" namespace paimon { +int64_t DataEvolutionSplitRead::BlobBunch::RowCount() const { + if (files_.empty()) { + return 0; + } + if (!has_row_ids_selection_) { + // Add enforces the union range to be contiguous + return union_end_row_id_ - union_first_row_id_; + } + // with a row-ids selection the scan may have pruned files, leaving holes in the union + int64_t row_count = 0; + for (const auto& range : Range::SortAndMergeOverlap(ranges_, /*adjacent=*/true)) { + row_count += range.Count(); + } + return row_count; +} + Status DataEvolutionSplitRead::BlobBunch::Add(const std::shared_ptr& file) { if (!BlobUtils::IsBlobFile(file->file_name)) { return Status::Invalid("Only blob file can be added to a blob bunch."); } PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); - if (first_row_id == latest_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with same first row id should have decreasing sequence number."); - } - // for files with the same first row id, file with larger sequence_number will be chosen, - // other files will be skipped - return Status::OK(); - } if (!files_.empty()) { - if (has_row_ids_selection_) { - // for the case: - // snapshot 1: blob0 [0, 9] - // snapshot 2: blob1 [0, 4] + blob2 [5, 9] - // when selected row id is {5}, only blob0 and blob2 is reserved in scan process, as - // blob1 has no intersect with {5} - // BlobBunch will first add blob0 [0, 9] - // then when it comes to blob2 [5, 9], blob0 will be removed as it has smaller sequence - // number - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number > latest_max_sequence_number_) { - row_count_ -= files_.back()->row_count; - files_.pop_back(); - } else { - return Status::OK(); - } - } - } else { - if (first_row_id < expected_next_first_row_id_) { - if (file->max_sequence_number >= latest_max_sequence_number_) { - return Status::Invalid( - "Blob file with overlapping row id should have decreasing sequence " - "number."); - } - // for files with overlapping, if the file with smaller sequence_number is chosen, - // there will not exist file with larger sequence_number - return Status::OK(); - } else if (first_row_id > expected_next_first_row_id_) { - return Status::Invalid( - fmt::format("Blob file first row id should be continuous, expect {} but got {}", - expected_next_first_row_id_, first_row_id)); - } + if (file->schema_id != files_[0]->schema_id) { + return Status::Invalid("All files in a blob bunch should have the same schema id."); } - if (!files_.empty()) { - if (file->schema_id != files_[0]->schema_id) { - return Status::Invalid("All files in a blob bunch should have the same schema id."); - } - if (file->write_cols != files_[0]->write_cols) { - return Status::Invalid( - "All files in a blob bunch should have the same write columns."); - } + if (file->write_cols != files_[0]->write_cols) { + return Status::Invalid("All files in a blob bunch should have the same write columns."); } } - row_count_ += file->row_count; - if (row_count_ > expected_row_count_) { + // files sharing a max sequence number form one layer, whose row id ranges must be disjoint; + // overlaps across layers are the expected shape of partial updates and are kept for the + // row-level placeholder fallback + auto [layer_iter, layer_inserted] = + sequence_group_end_.try_emplace(file->max_sequence_number, 0); + if (!layer_inserted && first_row_id < layer_iter->second) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping row id " + "ranges: file {} (max sequence number {}) starts at row id {} before the previous " + "file's end {}", + file->file_name, file->max_sequence_number, first_row_id, layer_iter->second)); + } + if (!has_row_ids_selection_ && !files_.empty() && first_row_id > union_end_row_id_) { + // a hole no layer covers cannot be aligned with the data files return Status::Invalid( - fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + fmt::format("Blob file first row id should be continuous, expect {} but got {}", + union_end_row_id_, first_row_id)); } + int64_t end_row_id = first_row_id + file->row_count; + layer_iter->second = end_row_id; + union_first_row_id_ = std::min(union_first_row_id_, first_row_id); + union_end_row_id_ = std::max(union_end_row_id_, end_row_id); + ranges_.emplace_back(first_row_id, end_row_id - 1); files_.push_back(file); - latest_max_sequence_number_ = file->max_sequence_number; - latest_first_row_id_ = first_row_id; - expected_next_first_row_id_ = latest_first_row_id_ + file->row_count; + if (!has_row_ids_selection_ && expected_row_count_ >= 0 && RowCount() > expected_row_count_) { + return Status::Invalid( + fmt::format("Blob files row count exceed the expect {}", expected_row_count_)); + } return Status::OK(); } DataEvolutionSplitRead::DataEvolutionSplitRead( @@ -235,7 +229,8 @@ Result> DataEvolutionSplitRead::CreateBlobViewReade std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), data_files, blob_view_schema, /*predicate=*/nullptr, /*dv_factory=*/nullptr, - /*row_ranges=*/std::nullopt, data_file_path_factory)); + /*row_ranges=*/std::nullopt, data_file_path_factory, + /*extra_format_options=*/{})); auto batch_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -314,7 +309,8 @@ Result> DataEvolutionSplitRead::InnerCreateReader( std::vector> raw_file_readers, CreateRawFileReaders(split_impl->Partition(), need_merge_files, raw_read_schema_, /*predicate=*/nullptr, - /*dv_factory=*/nullptr, row_ranges, data_file_path_factory)); + /*dv_factory=*/nullptr, row_ranges, data_file_path_factory, + /*extra_format_options=*/{})); assert(raw_file_readers.size() == 1); sub_readers.push_back(std::move(raw_file_readers[0])); } else { @@ -490,18 +486,30 @@ Result> DataEvolutionSplitRead::CreateU if (!read_fields_in_file.empty()) { // create new FieldMappingReader for read partial fields auto file_read_schema = DataField::ConvertDataFieldsToArrowSchema(read_fields_in_file); - PAIMON_ASSIGN_OR_RAISE(std::vector> file_readers, - CreateRawFileReaders(partition, bunch->Files(), file_read_schema, - /*predicate=*/nullptr, /*dv_factory=*/{}, - row_ranges, data_file_path_factory)); - if (file_readers.size() == 1) { - file_batch_readers[file_idx] = std::move(file_readers[0]); + auto blob_bunch = std::dynamic_pointer_cast(bunch); + if (blob_bunch && !blob_bunch->SequentialReadOptimize()) { + // blob files span multiple max sequence number layers: placeholder entries of + // newer layers must fall back row by row to older layers + PAIMON_ASSIGN_OR_RAISE( + file_batch_readers[file_idx], + CreateBlobFallbackReader(partition, blob_bunch->Files(), file_read_schema, + row_ranges, data_file_path_factory)); } else { - auto raw_readers = - ObjectUtils::MoveVector>(std::move(file_readers)); - // Concat multiple blob files that map to the same data file. - file_batch_readers[file_idx] = - std::make_unique(std::move(raw_readers), pool_); + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, bunch->Files(), file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, + /*extra_format_options=*/{})); + if (file_readers.size() == 1) { + file_batch_readers[file_idx] = std::move(file_readers[0]); + } else { + auto raw_readers = ObjectUtils::MoveVector>( + std::move(file_readers)); + // Concat multiple blob files that map to the same data file. + file_batch_readers[file_idx] = + std::make_unique(std::move(raw_readers), pool_); + } } } } @@ -511,6 +519,98 @@ Result> DataEvolutionSplitRead::CreateU field_offsets, pool_); } +namespace { +/// Selected row ids in [from, to] as sorted disjoint ranges: the whole range without a +/// selection, otherwise its intersection with the (possibly overlapping) selected ranges. +std::vector SelectedRangesInRange(int64_t from, int64_t to, + const std::optional>& row_ranges) { + if (!row_ranges) { + return {Range(from, to)}; + } + std::vector selected; + Range gap_range(from, to); + for (const auto& range : Range::SortAndMergeOverlap(row_ranges.value(), /*adjacent=*/true)) { + std::optional intersection = Range::Intersection(gap_range, range); + if (intersection) { + selected.push_back(*intersection); + } + } + return selected; +} +} // namespace + +Result> DataEvolutionSplitRead::CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const { + int64_t union_first_row_id = std::numeric_limits::max(); + int64_t union_last_row_id = std::numeric_limits::min(); + using FileWithFirstRowId = std::pair>; + std::map, std::greater<>> sequence_groups; + for (const auto& file : files) { + PAIMON_ASSIGN_OR_RAISE(int64_t first_row_id, file->NonNullFirstRowId()); + union_first_row_id = std::min(union_first_row_id, first_row_id); + union_last_row_id = std::max(union_last_row_id, first_row_id + file->row_count - 1); + sequence_groups[file->max_sequence_number].emplace_back(first_row_id, file); + } + // the blob format reader must emit placeholder sentinels instead of failing on them + const std::map blob_format_options = { + {BlobDefs::kEmitPlaceholderSentinelKey, "true"}}; + std::vector> groups; + groups.reserve(sequence_groups.size()); + for (auto& [max_sequence_number, group_files] : sequence_groups) { + std::stable_sort(group_files.begin(), group_files.end(), + [](const FileWithFirstRowId& f1, const FileWithFirstRowId& f2) { + return f1.first < f2.first; + }); + std::vector segments; + // pad row ids this layer does not cover with placeholder gaps, so that every layer spans + // the same union range and the groups can be stepped in lockstep + int64_t next_row_id = union_first_row_id; + for (const auto& [first_row_id, file] : group_files) { + if (first_row_id < next_row_id) { + return Status::Invalid(fmt::format( + "Blob files with the same max sequence number should not have overlapping " + "row id ranges: file {} (max sequence number {}) starts at row id {} before " + "the previous file's end {}", + file->file_name, max_sequence_number, first_row_id, next_row_id)); + } + if (first_row_id > next_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, first_row_id - 1, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + PAIMON_ASSIGN_OR_RAISE( + std::vector> file_readers, + CreateRawFileReaders(partition, {file}, file_read_schema, + /*predicate=*/nullptr, /*dv_factory=*/{}, row_ranges, + data_file_path_factory, blob_format_options)); + if (file_readers.size() != 1) { + return Status::Invalid("Unexpected: blob fallback file reader was skipped."); + } + segments.push_back(BlobFallbackBatchReader::Segment{std::move(file_readers[0]), {}}); + next_row_id = first_row_id + file->row_count; + } + if (next_row_id <= union_last_row_id) { + std::vector gap_selected_ranges = + SelectedRangesInRange(next_row_id, union_last_row_id, row_ranges); + if (!gap_selected_ranges.empty()) { + segments.push_back( + BlobFallbackBatchReader::Segment{nullptr, std::move(gap_selected_ranges)}); + } + } + groups.push_back(std::move(segments)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr fallback_reader, + BlobFallbackBatchReader::Create(std::move(groups), file_read_schema, + options_.GetReadBatchSize(), pool_)); + return std::move(fallback_reader); +} + Result DataEvolutionSplitRead::Match(const std::shared_ptr& split, bool force_keep_delete) const { return true; diff --git a/src/paimon/core/operation/data_evolution_split_read.h b/src/paimon/core/operation/data_evolution_split_read.h index c3ed5033f..0ca800604 100644 --- a/src/paimon/core/operation/data_evolution_split_read.h +++ b/src/paimon/core/operation/data_evolution_split_read.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include #include #include @@ -107,26 +109,40 @@ class DataEvolutionSplitRead : public AbstractSplitRead { std::vector> data_files_; }; + /// All blob files of one blob field in a merge group. Unlike data files, every file is kept + /// (aligned with Java's BlobFileBunch): files whose max sequence numbers differ form layers + /// of a data-evolution partial update, where newer layers record untouched rows as + /// placeholder entries and reading falls back row by row to older layers. + /// Files must be added ordered by first row id ascending (then max sequence number + /// descending), as produced by MergeRangesAndSort. class BlobBunch : public FieldBunch { public: explicit BlobBunch(int64_t expected_row_count, bool has_row_ids_selection) : expected_row_count_(expected_row_count), has_row_ids_selection_(has_row_ids_selection) {} - int64_t RowCount() const override { - return row_count_; - } + /// Number of distinct row ids covered by the added files. Without a row-ids selection + /// the covered range is contiguous (enforced by Add) and matches the data files. + int64_t RowCount() const override; const std::vector>& Files() const override { return files_; } Status Add(const std::shared_ptr& file); + /// True when every file shares one max sequence number: no placeholder entry can need + /// resolving, so the files can be read sequentially without the fallback merge. + bool SequentialReadOptimize() const { + return sequence_group_end_.size() <= 1; + } private: int64_t expected_row_count_ = -1; - int64_t latest_first_row_id_ = -1; - int64_t expected_next_first_row_id_ = -1; - int64_t latest_max_sequence_number_ = -1; - int64_t row_count_ = 0; bool has_row_ids_selection_ = false; + int64_t union_first_row_id_ = std::numeric_limits::max(); + /// Exclusive end of the union row id range covered so far. + int64_t union_end_row_id_ = std::numeric_limits::min(); + /// Per max sequence number: exclusive end of the last added range, to reject + /// overlapping files within one layer. + std::map sequence_group_end_; + std::vector ranges_; std::vector> files_; }; @@ -164,6 +180,16 @@ class DataEvolutionSplitRead : public AbstractSplitRead { const std::vector>& need_merge_files, const std::optional>& row_ranges, const std::shared_ptr& data_file_path_factory) const; + + /// Builds the row-level fallback reader for a blob bunch spanning multiple max sequence + /// number layers: groups the files by max sequence number, pads uncovered row id ranges of + /// each layer with placeholder gap segments, and resolves each row to the newest + /// non-placeholder layer. See BlobFallbackBatchReader. + Result> CreateBlobFallbackReader( + const BinaryRow& partition, const std::vector>& files, + const std::shared_ptr& file_read_schema, + const std::optional>& row_ranges, + const std::shared_ptr& data_file_path_factory) const; }; } // namespace paimon diff --git a/src/paimon/core/operation/data_evolution_split_read_test.cpp b/src/paimon/core/operation/data_evolution_split_read_test.cpp index aa6f9c9e1..4cf3a664b 100644 --- a/src/paimon/core/operation/data_evolution_split_read_test.cpp +++ b/src/paimon/core/operation/data_evolution_split_read_test.cpp @@ -145,72 +145,88 @@ TEST_F(DataEvolutionSplitReadTest, TestAddNonBlobFileInvalid) { TEST_F(DataEvolutionSplitReadTest, TestAddBlobWithSameFirstRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/1, + /*max_sequence_number=*/3, /*write_cols=*/std::optional>({"blob_col"})); - auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + auto blob_full_tail = + CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); + auto blob_short_tail = + CreateBlobFile("blob3", /*first_row_id=*/0, /*row_count=*/50, + /*max_sequence_number=*/1, + /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), - "Blob file with same first row id should have decreasing sequence number."); + // Files with the same first row id and lower sequence numbers are older layers of a + // partial update; they are kept for the row-level placeholder fallback, whether they + // cover the same range or only a shorter prefix of it. + ASSERT_OK(blob_bunch->Add(blob_full_tail)); + ASSERT_OK(blob_bunch->Add(blob_short_tail)); + + ASSERT_EQ(blob_bunch->Files().size(), 3); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_full_tail); + ASSERT_EQ(blob_bunch->Files()[2], blob_short_tail); + ASSERT_EQ(blob_bunch->RowCount(), 100); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithSameFirstRowIdAndLowerSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = - CreateBlobFile("blob2", /*first_row_id=*/0, /*row_count=*/50, + CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with same firstRowId and lower sequence number should be ignored + // Overlapping layers with different sequence numbers are kept for the fallback. ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowId) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/1, + /*max_sequence_number=*/2, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - // Adding file with overlapping row id and lower sequence number should be ignored ASSERT_OK(blob_bunch->Add(blob_tail)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->RowCount(), 200); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } -TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdAndHigherSequenceNumber) { +TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithOverlappingRowIdInSameLayer) { auto blob_entry = CreateBlobFile("blob1", /*first_row_id=*/0, /*row_count=*/100, /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_tail = CreateBlobFile("blob2", /*first_row_id=*/50, /*row_count=*/150, - /*max_sequence_number=*/2, + /*max_sequence_number=*/1, /*write_cols=*/std::optional>({"blob_col"})); auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/false); ASSERT_OK(blob_bunch->Add(blob_entry)); - ASSERT_NOK_WITH_MSG( - blob_bunch->Add(blob_tail), - "Blob file with overlapping row id should have decreasing sequence number."); + // Files sharing a max sequence number form one layer and must not overlap. + ASSERT_NOK_WITH_MSG(blob_bunch->Add(blob_tail), + "Blob files with the same max sequence number should not have overlapping " + "row id ranges"); } TEST_F(DataEvolutionSplitReadTest, TestAddBlobFileWithNonContinuousRowId) { @@ -265,12 +281,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, blob_entry is removed + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is a newer + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); - ASSERT_EQ(blob_bunch->Files()[0], blob_sub2); - ASSERT_EQ(blob_bunch->RowCount(), 5); + ASSERT_EQ(blob_bunch->Files().size(), 2); + ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); + ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { @@ -291,13 +309,14 @@ TEST_F(DataEvolutionSplitReadTest, TestRowIdSelectionWithOverlap2) { auto blob_bunch = std::make_shared( INT64_MAX, /*has_row_ids_selection=*/true); ASSERT_OK(blob_bunch->Add(blob_entry)); - // blob_sub1 will not be added, for it has been skipped by row_ids in scan process. - // after blob_sub2 is added, as blob_sub2 has smaller sequence number, blob_sub2 will be - // skipped. + // blob_sub1 was pruned by the row-ids selection in the scan process; blob_sub2 is an older + // layer and both files are kept for the row-level placeholder fallback. ASSERT_OK(blob_bunch->Add(blob_sub2)); - ASSERT_EQ(blob_bunch->Files().size(), 1); + ASSERT_EQ(blob_bunch->Files().size(), 2); ASSERT_EQ(blob_bunch->Files()[0], blob_entry); + ASSERT_EQ(blob_bunch->Files()[1], blob_sub2); ASSERT_EQ(blob_bunch->RowCount(), 10); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestRowIdSelection) { @@ -420,15 +439,15 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { std::vector> batch = batches[0]; ASSERT_EQ(batch.size(), 10); ASSERT_EQ(batch[0], data); - ASSERT_EQ(batch[1], blob_entry5); // pick - ASSERT_EQ(batch[2], blob_entry2); // skip - ASSERT_EQ(batch[3], blob_entry1); // skip - ASSERT_EQ(batch[4], blob_entry9); // pick - ASSERT_EQ(batch[5], blob_entry6); // skip - ASSERT_EQ(batch[6], blob_entry3); // skip - ASSERT_EQ(batch[7], blob_entry7); // pick - ASSERT_EQ(batch[8], blob_entry4); // skip - ASSERT_EQ(batch[9], blob_entry8); // pick + ASSERT_EQ(batch[1], blob_entry5); + ASSERT_EQ(batch[2], blob_entry2); + ASSERT_EQ(batch[3], blob_entry1); + ASSERT_EQ(batch[4], blob_entry9); + ASSERT_EQ(batch[5], blob_entry6); + ASSERT_EQ(batch[6], blob_entry3); + ASSERT_EQ(batch[7], blob_entry7); + ASSERT_EQ(batch[8], blob_entry4); + ASSERT_EQ(batch[9], blob_entry8); auto blob_field_to_field_id = [](const std::shared_ptr&) -> Result { return 0; @@ -440,12 +459,19 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario2) { ASSERT_EQ(bunch.size(), 2); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { @@ -561,20 +587,33 @@ TEST_F(DataEvolutionSplitReadTest, TestComplexBlobBunchScenario3) { ASSERT_EQ(bunch.size(), 3); auto blob_bunch = std::dynamic_pointer_cast(bunch[1]); - ASSERT_EQ(blob_bunch->Files().size(), 4); + // every sequence layer is kept for the row-level placeholder fallback + ASSERT_EQ(blob_bunch->Files().size(), 9); ASSERT_EQ(blob_bunch->Files()[0], blob_entry5); - ASSERT_EQ(blob_bunch->Files()[1], blob_entry9); - ASSERT_EQ(blob_bunch->Files()[2], blob_entry7); - ASSERT_EQ(blob_bunch->Files()[3], blob_entry8); + ASSERT_EQ(blob_bunch->Files()[1], blob_entry2); + ASSERT_EQ(blob_bunch->Files()[2], blob_entry1); + ASSERT_EQ(blob_bunch->Files()[3], blob_entry9); + ASSERT_EQ(blob_bunch->Files()[4], blob_entry6); + ASSERT_EQ(blob_bunch->Files()[5], blob_entry3); + ASSERT_EQ(blob_bunch->Files()[6], blob_entry7); + ASSERT_EQ(blob_bunch->Files()[7], blob_entry4); + ASSERT_EQ(blob_bunch->Files()[8], blob_entry8); ASSERT_EQ(blob_bunch->RowCount(), 1000); + ASSERT_FALSE(blob_bunch->SequentialReadOptimize()); auto blob_bunch2 = std::dynamic_pointer_cast(bunch[2]); - ASSERT_EQ(blob_bunch2->Files().size(), 4); + ASSERT_EQ(blob_bunch2->Files().size(), 9); ASSERT_EQ(blob_bunch2->Files()[0], blob_entry15); - ASSERT_EQ(blob_bunch2->Files()[1], blob_entry19); - ASSERT_EQ(blob_bunch2->Files()[2], blob_entry17); - ASSERT_EQ(blob_bunch2->Files()[3], blob_entry18); + ASSERT_EQ(blob_bunch2->Files()[1], blob_entry12); + ASSERT_EQ(blob_bunch2->Files()[2], blob_entry11); + ASSERT_EQ(blob_bunch2->Files()[3], blob_entry19); + ASSERT_EQ(blob_bunch2->Files()[4], blob_entry16); + ASSERT_EQ(blob_bunch2->Files()[5], blob_entry13); + ASSERT_EQ(blob_bunch2->Files()[6], blob_entry17); + ASSERT_EQ(blob_bunch2->Files()[7], blob_entry14); + ASSERT_EQ(blob_bunch2->Files()[8], blob_entry18); ASSERT_EQ(blob_bunch2->RowCount(), 1000); + ASSERT_FALSE(blob_bunch2->SequentialReadOptimize()); } TEST_F(DataEvolutionSplitReadTest, TestDifferentRowIdRange) { diff --git a/src/paimon/core/operation/merge_file_split_read.cpp b/src/paimon/core/operation/merge_file_split_read.cpp index 86df4841d..e1cee5a08 100644 --- a/src/paimon/core/operation/merge_file_split_read.cpp +++ b/src/paimon/core/operation/merge_file_split_read.cpp @@ -272,7 +272,8 @@ Result> MergeFileSplitRead::CreateNoMergeReader( std::vector> raw_file_readers, CreateRawFileReaders(data_split->Partition(), data_split->DataFiles(), read_schema, only_filter_key ? predicate_for_keys_ : context_->GetPredicate(), - dv_factory, /*row_ranges=*/{}, data_file_path_factory)); + dv_factory, /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); @@ -496,7 +497,8 @@ Result> MergeFileSplitRead::CreateReaderFo PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); assert(data_files.size() == raw_file_readers.size()); // KeyValueDataFileRecordReader converts arrow array from format reader to KeyValue objects diff --git a/src/paimon/core/operation/raw_file_split_read.cpp b/src/paimon/core/operation/raw_file_split_read.cpp index ff213a4ea..c027e1c72 100644 --- a/src/paimon/core/operation/raw_file_split_read.cpp +++ b/src/paimon/core/operation/raw_file_split_read.cpp @@ -81,7 +81,8 @@ Result> RawFileSplitRead::CreateReader( PAIMON_ASSIGN_OR_RAISE( std::vector> raw_file_readers, CreateRawFileReaders(partition, data_files, raw_read_schema_, predicate, dv_factory, - /*row_ranges=*/{}, data_file_path_factory)); + /*row_ranges=*/{}, data_file_path_factory, + /*extra_format_options=*/{})); auto raw_readers = ObjectUtils::MoveVector>(std::move(raw_file_readers)); diff --git a/src/paimon/format/blob/blob_file_batch_reader.cpp b/src/paimon/format/blob/blob_file_batch_reader.cpp index d5b5a2fe2..feb309cdd 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader.cpp @@ -18,6 +18,7 @@ #include #include +#include #include "arrow/api.h" #include "arrow/array/builder_dict.h" @@ -38,7 +39,7 @@ namespace paimon::blob { Result> BlobFileBatchReader::Create( const std::shared_ptr& input_stream, int32_t batch_size, bool blob_as_descriptor, - const std::shared_ptr& pool) { + bool emit_placeholder_sentinel, const std::shared_ptr& pool) { if (input_stream == nullptr) { return Status::Invalid("blob file batch reader create failed: input stream is nullptr"); } @@ -89,8 +90,9 @@ Result> BlobFileBatchReader::Create( } } PAIMON_ASSIGN_OR_RAISE(std::string file_path, input_stream->GetUri()); - auto reader = std::unique_ptr(new BlobFileBatchReader( - input_stream, file_path, blob_lengths, blob_offsets, batch_size, blob_as_descriptor, pool)); + auto reader = std::unique_ptr( + new BlobFileBatchReader(input_stream, file_path, blob_lengths, blob_offsets, batch_size, + blob_as_descriptor, emit_placeholder_sentinel, pool)); return reader; } @@ -99,6 +101,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, bool blob_as_descriptor, + bool emit_placeholder_sentinel, const std::shared_ptr& pool) : input_stream_(input_stream), file_path_(file_path), @@ -108,6 +111,7 @@ BlobFileBatchReader::BlobFileBatchReader(const std::shared_ptr& inp target_blob_offsets_(blob_offsets), batch_size_(batch_size), blob_as_descriptor_(blob_as_descriptor), + emit_placeholder_sentinel_(emit_placeholder_sentinel), pool_(pool), arrow_pool_(GetArrowPool(pool_)), metrics_(std::make_shared()) { @@ -168,11 +172,8 @@ Result> BlobFileBatchReader::NextBlobOffsets( PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(0)); int64_t data_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; // Null blobs contribute zero bytes to content - if (!IsTargetNull(i)) { - data_length += GetTargetContentLength(i); - } + data_length += GetTargetOutputLength(current_pos_ + k); PAIMON_RETURN_NOT_OK_FROM_ARROW(buffer_builder.Append(data_length)); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr offset_buffer, @@ -184,10 +185,7 @@ Result> BlobFileBatchReader::NextBlobContents( int32_t rows_to_read) const { int64_t total_length = 0; for (int32_t k = 0; k < rows_to_read; ++k) { - const size_t i = current_pos_ + k; - if (!IsTargetNull(i)) { - total_length += GetTargetContentLength(i); - } + total_length += GetTargetOutputLength(current_pos_ + k); } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr data_buffer, arrow::AllocateBuffer(total_length, arrow_pool_.get())); @@ -205,6 +203,46 @@ Result> BlobFileBatchReader::NextBlobContents( return data_buffer; } +Result> BlobFileBatchReader::BuildSentinelAwareContentArray( + int32_t rows_to_read) const { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::unique_ptr array_builder, + arrow::MakeBuilder(target_type_, arrow_pool_.get())); + auto builder = dynamic_cast(array_builder.get()); + if (builder == nullptr) { + return Status::Invalid("cast to struct builder failed"); + } + auto field_builder = dynamic_cast(builder->field_builder(0)); + if (field_builder == nullptr) { + return Status::Invalid("cast to large binary builder failed"); + } + std::string content; + for (int32_t k = 0; k < rows_to_read; ++k) { + const size_t i = current_pos_ + k; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); + if (IsTargetNull(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + continue; + } + if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); + continue; + } + int64_t length = GetTargetContentLength(i); + content.resize(length); + PAIMON_RETURN_NOT_OK(ReadBlobContentAt(GetTargetContentOffset(i), length, + reinterpret_cast(content.data()))); + if (BlobDefs::HasPlaceholderSentinelPrefix(content.data(), content.size())) { + // escape so the fallback merge cannot mistake the value for a placeholder + content.insert(0, BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength); + } + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append(content.data(), content.size())); + } + std::shared_ptr array; + PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array)); + return array; +} + Result> BlobFileBatchReader::BuildNullBitmap( int32_t rows_to_read) const { bool has_null = false; @@ -250,6 +288,9 @@ Result> BlobFileBatchReader::BuildTargetArray( int32_t rows_to_read) const { std::shared_ptr blob_array; if (!blob_as_descriptor_) { + if (emit_placeholder_sentinel_) { + return BuildSentinelAwareContentArray(rows_to_read); + } return BuildContentArray(rows_to_read); } // For descriptor mode, build using StructBuilder to handle nulls properly @@ -268,6 +309,9 @@ Result> BlobFileBatchReader::BuildTargetArray( PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Append()); if (IsTargetNull(i)) { PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->AppendNull()); + } else if (IsTargetPlaceholder(i)) { + PAIMON_RETURN_NOT_OK_FROM_ARROW(field_builder->Append( + BlobDefs::kPlaceholderSentinel, BlobDefs::kPlaceholderSentinelLength)); } else { int64_t offset = GetTargetContentOffset(i); int64_t length = GetTargetContentLength(i); @@ -297,6 +341,17 @@ Result BlobFileBatchReader::NextBatch() { } int32_t left_rows = target_blob_lengths_.size() - current_pos_; int32_t rows_to_read = std::min(left_rows, batch_size_); + if (!emit_placeholder_sentinel_) { + for (int32_t k = 0; k < rows_to_read; ++k) { + if (IsTargetPlaceholder(current_pos_ + k)) { + return Status::Invalid(fmt::format( + "blob file {} contains a placeholder entry (bin_length {}) written by a " + "data-evolution partial update; it can only be resolved by the data-evolution " + "blob fallback read path", + file_path_, BlobDefs::kPlaceholderBinLength)); + } + } + } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr blob_array, BuildTargetArray(rows_to_read)); std::unique_ptr c_array = std::make_unique(); diff --git a/src/paimon/format/blob/blob_file_batch_reader.h b/src/paimon/format/blob/blob_file_batch_reader.h index 27549c190..a01bf42c8 100644 --- a/src/paimon/format/blob/blob_file_batch_reader.h +++ b/src/paimon/format/blob/blob_file_batch_reader.h @@ -87,9 +87,17 @@ namespace paimon::blob { /// - Current version is 1. class BlobFileBatchReader : public FileBatchReader { public: + /// `emit_placeholder_sentinel` controls how placeholder entries (bin_length == + /// BlobDefs::kPlaceholderBinLength) are read: when false they fail the read, as resolving + /// them requires the data-evolution blob fallback path; when true they are returned as the + /// non-null BlobDefs::kPlaceholderSentinel bytes for that path to merge away, and any + /// stored value starting with the sentinel bytes is escaped by prepending one sentinel + /// copy, so real values can never be mistaken for placeholders (the fallback merge strips + /// the escape). static Result> Create( const std::shared_ptr& input_stream, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Result> GetFileSchema() const override; @@ -144,7 +152,8 @@ class BlobFileBatchReader : public FileBatchReader { BlobFileBatchReader(const std::shared_ptr& input_stream, const std::string& file_path, const std::vector& blob_lengths, const std::vector& blob_offsets, int32_t batch_size, - bool blob_as_descriptor, const std::shared_ptr& pool); + bool blob_as_descriptor, bool emit_placeholder_sentinel, + const std::shared_ptr& pool); Status ReadBlobContentAt(const int64_t offset, const int64_t length, uint8_t* content) const; @@ -153,6 +162,11 @@ class BlobFileBatchReader : public FileBatchReader { /// Builds a null bitmap buffer for the given rows. Returns nullptr if no nulls. Result> BuildNullBitmap(int32_t rows_to_read) const; Result> BuildContentArray(int32_t rows_to_read) const; + /// Raw-content variant of BuildContentArray for placeholder-aware reads: placeholder + /// entries become the sentinel bytes and stored values starting with the sentinel are + /// escaped by prepending one sentinel copy. + Result> BuildSentinelAwareContentArray( + int32_t rows_to_read) const; Result> BuildTargetArray(int32_t rows_to_read) const; /// Returns true if the blob at the given index is null (bin_length == kNullBinLength). @@ -160,6 +174,20 @@ class BlobFileBatchReader : public FileBatchReader { return target_blob_lengths_[index] == BlobDefs::kNullBinLength; } + bool IsTargetPlaceholder(size_t index) const { + return target_blob_lengths_[index] == BlobDefs::kPlaceholderBinLength; + } + + /// Content bytes the blob at the given index contributes to the output buffer. Only used + /// by the fast content path, which never sees placeholder entries: strict mode fails on + /// them first and placeholder-aware mode goes through BuildSentinelAwareContentArray. + int64_t GetTargetOutputLength(size_t index) const { + if (IsTargetNull(index)) { + return 0; + } + return GetTargetContentLength(index); + } + int64_t GetTargetContentOffset(size_t index) const { return target_blob_offsets_[index] + BlobDefs::kContentStartOffset; } @@ -179,6 +207,7 @@ class BlobFileBatchReader : public FileBatchReader { const int32_t batch_size_; const bool blob_as_descriptor_; + const bool emit_placeholder_sentinel_; std::shared_ptr pool_; std::shared_ptr arrow_pool_; diff --git a/src/paimon/format/blob/blob_file_batch_reader_test.cpp b/src/paimon/format/blob/blob_file_batch_reader_test.cpp index 4e53dfb4c..ca524431d 100644 --- a/src/paimon/format/blob/blob_file_batch_reader_test.cpp +++ b/src/paimon/format/blob/blob_file_batch_reader_test.cpp @@ -162,9 +162,10 @@ TEST_F(BlobFileBatchReaderTest, TestRowNumbers) { ASSERT_OK_AND_ASSIGN( std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -213,9 +214,9 @@ TEST_F(BlobFileBatchReaderTest, InvalidScenario) { } { ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(/*input_stream=*/input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(/*input_stream=*/input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->GetFileSchema(), "blob file has no self-describing file schema"); ASSERT_TRUE(reader->GetReaderMetrics()); @@ -237,7 +238,8 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream, struct_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system, pool_)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -248,9 +250,10 @@ TEST_P(BlobFileBatchReaderTest, EmptyFile) { ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, file_system->Open(dir->Str() + "/file.blob")); - ASSERT_OK_AND_ASSIGN(auto reader, BlobFileBatchReader::Create( - input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + ASSERT_OK_AND_ASSIGN(auto reader, + BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_OK(reader->SetReadSchema(&c_schema, nullptr, std::nullopt)); ASSERT_OK_AND_ASSIGN(auto number_of_rows, reader->GetNumberOfRows()); @@ -273,9 +276,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(/*read_schema=*/nullptr, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "SetReadSchema failed: read schema cannot be nullptr"); @@ -298,9 +301,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "read schema field number 2 is not 1"); @@ -323,9 +326,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); ASSERT_NOK_WITH_MSG(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, /*selection_bitmap=*/std::nullopt), "field my_blob_field: large_binary is not BLOB"); @@ -346,9 +349,9 @@ TEST_F(BlobFileBatchReaderTest, SetReadSchemaWithInvalidInputs) { std::shared_ptr input_stream, fs->Open(table_path + "/bucket-0/data-d7816e8e-6c6d-4e28-9137-837cdf706350-1.blob")); ASSERT_OK_AND_ASSIGN( - auto reader, - BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1, /*blob_as_descriptor=*/true, pool_)); + auto reader, BlobFileBatchReader::Create(input_stream, + /*batch_size=*/1, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool_)); RoaringBitmap32 roaring; roaring.Add(0); roaring.Add(1); diff --git a/src/paimon/format/blob/blob_format_writer.cpp b/src/paimon/format/blob/blob_format_writer.cpp index 875566ffc..d1dd48812 100644 --- a/src/paimon/format/blob/blob_format_writer.cpp +++ b/src/paimon/format/blob/blob_format_writer.cpp @@ -39,7 +39,7 @@ namespace paimon::blob { BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, - bool write_null_on_fetch_failure, + bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) : out_(out), @@ -48,7 +48,8 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con fs_(fs), pool_(pool), write_null_on_missing_file_(write_null_on_missing_file), - write_null_on_fetch_failure_(write_null_on_fetch_failure) { + write_null_on_fetch_failure_(write_null_on_fetch_failure), + write_placeholder_(write_placeholder) { // Create() has already checked that data_type has exactly one BLOB field. blob_field_name_ = data_type_->field(0)->name(); metrics_ = std::make_shared(); @@ -59,7 +60,7 @@ BlobFormatWriter::BlobFormatWriter(const std::shared_ptr& out, con Result> BlobFormatWriter::Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool) { if (out == nullptr) { return Status::Invalid("blob format writer create failed. out is nullptr"); @@ -82,8 +83,9 @@ Result> BlobFormatWriter::Create( fmt::format("field {} is not BLOB", data_type->field(0)->ToString())); } PAIMON_ASSIGN_OR_RAISE(std::string uri, out->GetUri()); - return std::unique_ptr(new BlobFormatWriter( - out, uri, data_type, write_null_on_missing_file, write_null_on_fetch_failure, fs, pool)); + return std::unique_ptr( + new BlobFormatWriter(out, uri, data_type, write_null_on_missing_file, + write_null_on_fetch_failure, write_placeholder, fs, pool)); } Status BlobFormatWriter::AddBatch(ArrowArray* batch) { @@ -117,7 +119,20 @@ Status BlobFormatWriter::AddBatch(ArrowArray* batch) { const auto& blob_array = arrow::internal::checked_cast(*child_array); assert(blob_array.length() == 1); - PAIMON_RETURN_NOT_OK(WriteBlob(blob_array.GetView(0))); + std::string_view blob_data = blob_array.GetView(0); + // Only a data-evolution partial-update write interprets the placeholder protocol; any + // other write stores the bytes verbatim, so user data can never become a placeholder. + if (write_placeholder_ && + BlobDefs::HasPlaceholderSentinelPrefix(blob_data.data(), blob_data.size())) { + if (BlobDefs::IsPlaceholderSentinel(blob_data.data(), blob_data.size())) { + // row not touched by the partial update: record it as a placeholder entry + bin_lengths_.push_back(BlobDefs::kPlaceholderBinLength); + return Status::OK(); + } + // a real value escaped by the producer: strip the leading sentinel copy + blob_data.remove_prefix(BlobDefs::kPlaceholderSentinelLength); + } + PAIMON_RETURN_NOT_OK(WriteBlob(blob_data)); PAIMON_RETURN_NOT_OK(Flush()); return Status::OK(); } diff --git a/src/paimon/format/blob/blob_format_writer.h b/src/paimon/format/blob/blob_format_writer.h index 9dbe57a87..615d64c47 100644 --- a/src/paimon/format/blob/blob_format_writer.h +++ b/src/paimon/format/blob/blob_format_writer.h @@ -68,9 +68,16 @@ class BlobFormatWriter : public FormatWriter { /// `write_null_on_missing_file` is enabled; otherwise a missing file follows /// `write_null_on_fetch_failure` like any other failed open. /// See Options::BLOB_WRITE_NULL_ON_MISSING_FILE / BLOB_WRITE_NULL_ON_FETCH_FAILURE. + /// + /// `write_placeholder` (see BlobDefs::kWritePlaceholderKey, false unless the write is a + /// data-evolution partial update) applies the placeholder protocol to incoming values: a + /// value equal to BlobDefs::kPlaceholderSentinel is persisted as a placeholder entry + /// (bin_length -2, no data bytes), and a longer value starting with the sentinel is + /// unescaped by stripping the leading sentinel before writing. When disabled, values are + /// stored verbatim and can never be turned into placeholder entries. static Result> Create( const std::shared_ptr& out, const std::shared_ptr& data_type, - bool write_null_on_missing_file, bool write_null_on_fetch_failure, + bool write_null_on_missing_file, bool write_null_on_fetch_failure, bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status AddBatch(ArrowArray* batch) override; @@ -91,7 +98,7 @@ class BlobFormatWriter : public FormatWriter { BlobFormatWriter(const std::shared_ptr& out, const std::string& uri, const std::shared_ptr& data_type, bool write_null_on_missing_file, bool write_null_on_fetch_failure, - const std::shared_ptr& fs, + bool write_placeholder, const std::shared_ptr& fs, const std::shared_ptr& pool); Status WriteBlob(std::string_view blob_data); @@ -135,6 +142,7 @@ class BlobFormatWriter : public FormatWriter { std::shared_ptr metrics_; bool write_null_on_missing_file_ = false; bool write_null_on_fetch_failure_ = false; + bool write_placeholder_ = false; uint64_t null_on_missing_file_count_ = 0; uint64_t null_on_fetch_failure_count_ = 0; std::unique_ptr logger_; diff --git a/src/paimon/format/blob/blob_format_writer_test.cpp b/src/paimon/format/blob/blob_format_writer_test.cpp index d54f6042c..d067a7624 100644 --- a/src/paimon/format/blob/blob_format_writer_test.cpp +++ b/src/paimon/format/blob/blob_format_writer_test.cpp @@ -129,7 +129,16 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> CreateDefaultWriter() const { return BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_); + } + + /// Create a writer in placeholder mode, as used by data-evolution partial updates. + Result> CreatePlaceholderWriter() const { + return BlobFormatWriter::Create(output_stream_, struct_type_, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/true, file_system_, pool_); } Status AddBatchOnce(const std::shared_ptr& format_writer, @@ -161,9 +170,11 @@ class BlobFormatWriterTestBase : public ::testing::Test { Result> ReadBackAsData() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr input_stream, file_system_->Open(dir_->Str() + "/file.blob")); - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, - /*blob_as_descriptor=*/false, pool_)); + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*schema, &c_schema)); @@ -253,7 +264,8 @@ TEST_P(BlobFormatWriterTest, TestSimple) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -284,41 +296,47 @@ TEST_P(BlobFormatWriterTest, TestCreateWithInvalidParameters) { // Test with nullptr output stream ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(nullptr, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. out is nullptr"); // Test with nullptr data type ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, nullptr, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob format writer create failed. data_type is nullptr"); // Test with nullptr memory pool ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, nullptr), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, nullptr), "blob format writer create failed. pool is nullptr"); // Test with nullptr file system ASSERT_NOK_WITH_MSG( BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, nullptr, pool_), + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, nullptr, pool_), "blob format writer create failed. fs is nullptr"); // Test with invalid field count (more than 1 field) auto multi_field_type = arrow::struct_( {arrow::field("blob_col1", arrow::binary()), arrow::field("blob_col2", arrow::binary())}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, multi_field_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, multi_field_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "blob data type field number 2 is not 1"); // Test with non-blob field (missing blob metadata) auto non_blob_field = arrow::field("regular_col", arrow::binary()); auto non_blob_type = arrow::struct_({non_blob_field}); - ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create( - output_stream_, non_blob_type, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, file_system_, pool_), + ASSERT_NOK_WITH_MSG(BlobFormatWriter::Create(output_stream_, non_blob_type, + /*write_null_on_missing_file=*/false, + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_), "field regular_col: binary is not BLOB"); } @@ -443,7 +461,8 @@ TEST_P(BlobFormatWriterTest, TestLargeBlob) { file_system_->Open(dir_->Str() + "/file.blob")); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -492,7 +511,8 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithNullValues) { ASSERT_TRUE(input_stream); ASSERT_OK_AND_ASSIGN( std::unique_ptr reader, - BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, pool_)); + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, blob_as_descriptor_, + /*emit_placeholder_sentinel=*/false, pool_)); auto schema = arrow::schema(struct_type_->fields()); ::ArrowSchema c_schema; ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); @@ -525,7 +545,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnMissingFile) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, Blob::FromPath(dir_->Str() + "/not_exist_file", /*offset=*/0, @@ -563,7 +584,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnFetchFailure) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); std::string file = paimon::test::GetDataDir() + "/xxhash.data"; ASSERT_OK_AND_ASSIGN(std::shared_ptr bad_offset_blob, @@ -602,7 +624,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnBothOptionsEnabled) { ASSERT_OK_AND_ASSIGN( std::shared_ptr writer, BlobFormatWriter::Create(output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); // Row 0: missing file -> NULL. ASSERT_OK_AND_ASSIGN(std::shared_ptr missing_blob, @@ -667,7 +690,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), 0); } @@ -676,7 +700,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); } // The same fetch failure, now converted to NULL. @@ -684,7 +709,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); } // Missing file with only fetch-failure enabled: no existence check runs, so the file is @@ -695,7 +721,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, io_error_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, io_error_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, missing_array)); ASSERT_EQ(io_error_fs->OpenCallCount(), open_calls_before + 1); ASSERT_OK_AND_ASSIGN( @@ -713,7 +740,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); // One check before the open and one after it. ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); @@ -733,7 +761,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 2); ASSERT_OK_AND_ASSIGN( @@ -752,7 +781,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, existing_array)); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -762,7 +792,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullClassifiesByExistence) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/false, vanishing_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, vanishing_fs, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, existing_array), "mock io error"); ASSERT_EQ(vanishing_fs->ExistsCallCount(), 0); } @@ -785,14 +816,16 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnInvalidDescriptor) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, file_system_, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_NOK_WITH_MSG(AddBatchOnce(writer, array), "invalid blob descriptor"); } { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, file_system_, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, file_system_, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_OK(writer->Flush()); ASSERT_OK(writer->Finish()); @@ -820,7 +853,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 1); ASSERT_OK(writer->Flush()); @@ -850,7 +884,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/false, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/false, + /*write_placeholder=*/false, exists_fail_fs, pool_)); // The reported failure names the check and keeps the underlying status message. Status check_status = AddBatchOnce(writer, array); ASSERT_NOK_WITH_MSG(check_status, "failed to check existence of blob file"); @@ -864,7 +899,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( output_stream_, struct_type_, /*write_null_on_missing_file=*/true, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); // One check before the open and one after it failed. ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 2); @@ -884,7 +920,8 @@ TEST_F(BlobFormatWriterWriteNullTest, TestWriteNullOnExistsCheckFailure) { ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, BlobFormatWriter::Create( side_stream, struct_type_, /*write_null_on_missing_file=*/false, - /*write_null_on_fetch_failure=*/true, exists_fail_fs, pool_)); + /*write_null_on_fetch_failure=*/true, + /*write_placeholder=*/false, exists_fail_fs, pool_)); ASSERT_OK(AddBatchOnce(writer, array)); ASSERT_EQ(exists_fail_fs->ExistsCallCount(), 0); ASSERT_OK_AND_ASSIGN( @@ -931,4 +968,264 @@ TEST_P(BlobFormatWriterTest, TestAddBatchWithZeroLengthBlob) { ASSERT_EQ(buffer, expected); } +/// Placeholder tests always feed the sentinel bytes of the placeholder write protocol, so +/// they do not depend on the blob_as_descriptor_ parameter and run once on the +/// non-parameterized fixture. +using BlobFormatWriterPlaceholderTest = BlobFormatWriterTestBase; + +std::string PlaceholderSentinelBytes() { + return std::string(BlobDefs::PlaceholderSentinelView()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestWritePlaceholderGoldenBytes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + + // row 0: inline bytes "inline"; row 1: null; row 2: placeholder + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + + arrow::StructBuilder struct_builder(struct_type_, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(blob_builder->AppendNull().ok()); + std::shared_ptr null_array; + ASSERT_TRUE(struct_builder.Finish(&null_array).ok()); + ASSERT_OK(AddBatchOnce(writer, null_array)); + + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + // Verify byte-level alignment with the Java writer: null and placeholder rows occupy no + // data bytes; the index records [22, -1, -2] as zigzag varint deltas [0x2c, 0x2d, 0x01]. + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_TRUE(input_stream); + ASSERT_OK_AND_ASSIGN(int64_t file_length, input_stream->Length()); + ASSERT_EQ(file_length, 30); + std::vector buffer(file_length); + ASSERT_OK_AND_ASSIGN(auto read_length, + input_stream->Read(reinterpret_cast(buffer.data()), buffer.size())); + ASSERT_EQ(read_length, 30); + std::vector expected = {{// record 0: magic + "inline" + bin_length(22) + crc32 + 0xcf, 0x11, 0x4e, 0x58, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x60, + 0xc8, 0xe9, + // index of [22, -1, -2] + 0x2c, 0x2d, 0x01, + // footer: index length + version + 0x03, 0x00, 0x00, 0x00, 0x01}}; + ASSERT_EQ(buffer, expected); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderStrictAndAwareModes) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto inline_array, MakeBlobArrayFromBytes("inline")); + ASSERT_OK(AddBatchOnce(writer, inline_array)); + ASSERT_OK_AND_ASSIGN(auto placeholder_array, + MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, placeholder_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // default (strict) mode: reading a placeholder entry fails + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_NOK_WITH_MSG(reader->NextBatch(), "placeholder"); + } + + // placeholder-aware mode: the entry is returned as the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), "inline"); + ASSERT_FALSE(binary_array->IsNull(1)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } + + // placeholder-aware descriptor mode also returns the sentinel bytes + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(1), PlaceholderSentinelBytes()); + } +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestReadPlaceholderWithSelectionBitmap) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + ASSERT_OK_AND_ASSIGN(auto array0, MakeBlobArrayFromBytes("first")); + ASSERT_OK(AddBatchOnce(writer, array0)); + ASSERT_OK_AND_ASSIGN(auto array1, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, array1)); + ASSERT_OK_AND_ASSIGN(auto array2, MakeBlobArrayFromBytes("third")); + ASSERT_OK(AddBatchOnce(writer, array2)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + RoaringBitmap32 selection; + selection.Add(1); + selection.Add(2); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, selection)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 2); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); + ASSERT_EQ(binary_array->GetString(1), "third"); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestSentinelBytesVerbatimWithoutPlaceholderMode) { + // outside placeholder mode a user blob whose bytes equal the sentinel is a normal value: it + // must be stored as a real entry (not persisted as bin_length -2) and read back unchanged + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreateDefaultWriter()); + ASSERT_OK_AND_ASSIGN(auto sentinel_array, MakeBlobArrayFromBytes(PlaceholderSentinelBytes())); + ASSERT_OK(AddBatchOnce(writer, sentinel_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN(std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + auto schema = arrow::schema(struct_type_->fields()); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + ASSERT_EQ(struct_array->length(), 1); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_FALSE(binary_array->IsNull(0)); + ASSERT_EQ(binary_array->GetString(0), PlaceholderSentinelBytes()); +} + +TEST_F(BlobFormatWriterPlaceholderTest, TestEscapedSentinelRoundTrip) { + // in placeholder mode a real value starting with the sentinel must be escaped by the + // producer (one extra sentinel copy); the writer strips the escape and stores the original + // bytes, a strict read returns them verbatim, and a placeholder-aware read re-escapes them + // so the fallback merge cannot mistake them for placeholders + ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, CreatePlaceholderWriter()); + std::string sentinel = PlaceholderSentinelBytes(); + ASSERT_OK_AND_ASSIGN(auto escaped_sentinel_array, MakeBlobArrayFromBytes(sentinel + sentinel)); + ASSERT_OK(AddBatchOnce(writer, escaped_sentinel_array)); + ASSERT_OK_AND_ASSIGN(auto escaped_prefixed_array, + MakeBlobArrayFromBytes(sentinel + sentinel + "suffix")); + ASSERT_OK(AddBatchOnce(writer, escaped_prefixed_array)); + ASSERT_OK(writer->Flush()); + ASSERT_OK(writer->Finish()); + + auto schema = arrow::schema(struct_type_->fields()); + + // strict mode returns the original (unescaped) bytes verbatim + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/false, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + "suffix"); + } + + // placeholder-aware mode re-escapes the stored values with one extra sentinel copy + { + ASSERT_OK_AND_ASSIGN(std::shared_ptr input_stream, + file_system_->Open(dir_->Str() + "/file.blob")); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr reader, + BlobFileBatchReader::Create(input_stream, /*batch_size=*/1024, + /*blob_as_descriptor=*/false, + /*emit_placeholder_sentinel=*/true, pool_)); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(reader->SetReadSchema(&c_schema, /*predicate=*/nullptr, + /*selection_bitmap=*/std::nullopt)); + ASSERT_OK_AND_ASSIGN(auto chunked_array, + paimon::test::ReadResultCollector::CollectResult(reader.get())); + auto concat_array = arrow::Concatenate(chunked_array->chunks()).ValueOrDie(); + auto struct_array = arrow::internal::checked_pointer_cast(concat_array); + auto binary_array = + arrow::internal::checked_pointer_cast(struct_array->field(0)); + ASSERT_EQ(binary_array->GetString(0), sentinel + sentinel); + ASSERT_EQ(binary_array->GetString(1), sentinel + sentinel + "suffix"); + } +} + } // namespace paimon::blob::test diff --git a/src/paimon/format/blob/blob_reader_builder.h b/src/paimon/format/blob/blob_reader_builder.h index eb2ce9522..a5d20fbdd 100644 --- a/src/paimon/format/blob/blob_reader_builder.h +++ b/src/paimon/format/blob/blob_reader_builder.h @@ -43,7 +43,11 @@ class BlobReaderBuilder : public ReaderBuilder { PAIMON_ASSIGN_OR_RAISE( bool blob_as_descriptor, OptionsUtils::GetValueFromMap(options_, Options::BLOB_AS_DESCRIPTOR, false)); - return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, pool_); + PAIMON_ASSIGN_OR_RAISE(bool emit_placeholder_sentinel, + OptionsUtils::GetValueFromMap( + options_, BlobDefs::kEmitPlaceholderSentinelKey, false)); + return BlobFileBatchReader::Create(input_stream, batch_size_, blob_as_descriptor, + emit_placeholder_sentinel, pool_); } private: diff --git a/src/paimon/format/blob/blob_stats_extractor.cpp b/src/paimon/format/blob/blob_stats_extractor.cpp index ca5efe831..14445dfc6 100644 --- a/src/paimon/format/blob/blob_stats_extractor.cpp +++ b/src/paimon/format/blob/blob_stats_extractor.cpp @@ -51,7 +51,8 @@ BlobStatsExtractor::ExtractWithFileInfo(const std::shared_ptr& file_ PAIMON_ASSIGN_OR_RAISE( std::unique_ptr blob_reader, BlobFileBatchReader::Create(input_stream, - /*batch_size=*/1024, /*blob_as_descriptor=*/true, pool)); + /*batch_size=*/1024, /*blob_as_descriptor=*/true, + /*emit_placeholder_sentinel=*/false, pool)); ColumnStatsVector result_stats; result_stats.push_back( ColumnStats::CreateStringColumnStats(std::nullopt, std::nullopt, std::nullopt)); diff --git a/src/paimon/format/blob/blob_writer_builder.h b/src/paimon/format/blob/blob_writer_builder.h index 12caa0e16..7cc0b5927 100644 --- a/src/paimon/format/blob/blob_writer_builder.h +++ b/src/paimon/format/blob/blob_writer_builder.h @@ -24,6 +24,7 @@ #include #include "arrow/api.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/utils/options_utils.h" #include "paimon/defs.h" #include "paimon/format/blob/blob_format_writer.h" @@ -73,8 +74,11 @@ class BlobWriterBuilder : public SpecificFSWriterBuilder { PAIMON_ASSIGN_OR_RAISE(bool write_null_on_fetch_failure, OptionsUtils::GetValueFromMap( options_, Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE, false)); + PAIMON_ASSIGN_OR_RAISE( + bool write_placeholder, + OptionsUtils::GetValueFromMap(options_, BlobDefs::kWritePlaceholderKey, false)); return BlobFormatWriter::Create(out, data_type_, write_null_on_missing_file, - write_null_on_fetch_failure, fs_, pool_); + write_null_on_fetch_failure, write_placeholder, fs_, pool_); } private: diff --git a/test/inte/blob_table_inte_test.cpp b/test/inte/blob_table_inte_test.cpp index 3a8900099..590184632 100644 --- a/test/inte/blob_table_inte_test.cpp +++ b/test/inte/blob_table_inte_test.cpp @@ -42,6 +42,7 @@ #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_row_writer.h" +#include "paimon/common/data/blob_defs.h" #include "paimon/common/data/blob_descriptor.h" #include "paimon/common/data/blob_utils.h" #include "paimon/common/data/blob_view_struct.h" @@ -1045,6 +1046,486 @@ TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyWriteWithFirstRowId) { ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); } +/// Build a single-blob-column StructArray for a data-evolution partial update: "PH" marks a row +/// whose blob is not updated (persisted as a placeholder entry), std::nullopt a null blob. +std::shared_ptr MakeBlobUpdateArray( + const std::shared_ptr& blob_field, + const std::vector>& rows) { + auto struct_type = arrow::struct_({blob_field}); + arrow::StructBuilder struct_builder(struct_type, arrow::default_memory_pool(), + {std::make_shared()}); + auto blob_builder = static_cast(struct_builder.field_builder(0)); + for (const auto& row : rows) { + EXPECT_TRUE(struct_builder.Append().ok()); + if (!row) { + EXPECT_TRUE(blob_builder->AppendNull().ok()); + } else if (*row == "PH") { + std::string_view sentinel = BlobDefs::PlaceholderSentinelView(); + EXPECT_TRUE(blob_builder->Append(sentinel.data(), sentinel.size()).ok()); + } else { + EXPECT_TRUE(blob_builder->Append(row->data(), row->size()).ok()); + } + } + std::shared_ptr array; + EXPECT_TRUE(struct_builder.Finish(&array).ok()); + return std::dynamic_pointer_cast(array); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateFallback) { + // the blob column is updated to null below, so it must be nullable + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Initial full-row write assigns row ids 0-2. + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "blob_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // Partial update: only row 1 gets a new blob, the untouched rows are written as + // placeholder entries and must fall back to the previous blob file when read. + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_a"], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // updating a blob to null is not a placeholder: the null must win over older layers + auto null_update_array = MakeBlobUpdateArray(fields[2], {std::nullopt, "PH", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, + WriteArray(table_path, {}, {"b0"}, {null_update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array2 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", null], + [2, "b", "updated_b"], + [3, "c", "blob_c"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array2)); + + // row ids still come from the data files and stay aligned with the fallback result + auto expected_with_row_id = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({fields[0], fields[1], fields[2], SpecialFields::RowId().field_}), + R"([ + [1, "a", null, 0], + [2, "b", "updated_b", 1], + [3, "c", "blob_c", 2] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, {"f0", "f1", "b0", "_ROW_ID"}, expected_with_row_id)); + + // blob_as_descriptor read mode: the fallback-merged values come back as descriptors and + // must still resolve to the same bytes + ASSERT_OK_AND_ASSIGN(auto plan, ScanTable(table_path)); + std::map read_options = {{Options::BLOB_AS_DESCRIPTOR, "true"}}; + ASSERT_OK_AND_ASSIGN(auto desc_result, ReadTable(table_path, schema->field_names(), plan, + /*predicate=*/nullptr, read_options)); + ASSERT_TRUE(desc_result.chunked_array); + auto desc_concat = arrow::Concatenate(desc_result.chunked_array->chunks()).ValueOrDie(); + auto desc_struct = std::dynamic_pointer_cast(desc_concat); + ASSERT_TRUE(desc_struct); + ASSERT_OK_AND_ASSIGN(auto resolved, ConvertDescriptorToRawBlob(desc_struct, {"b0"})); + ASSERT_OK_AND_ASSIGN(auto expected_with_rk, PrependRowKindColumn(expected_array2)); + ASSERT_TRUE(resolved->Equals(expected_with_rk)) + << "result:" << resolved->ToString() << "\nexpected:" << expected_with_rk->ToString(); +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateMultipleLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto src_array0 = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "blob_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "blob_3"] + ])") + .ValueOrDie()); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_array0})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // second layer updates row 0 within rows [0, 1] + auto update_array1 = MakeBlobUpdateArray(fields[2], {"update1_0", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // third layer updates row 3 within rows [2, 3]; each layer only partially covers the + // range, the uncovered parts behave as placeholders + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "update2_3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "a", "update1_0"], + [2, "b", "blob_1"], + [3, "c", "blob_2"], + [4, "d", "update2_3"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array)); + + // row-range pushdown where the selected rows fall in one layer's file and in another + // layer's uncovered gap: row 1 is a gap row for the third layer, row 2 for the second + auto expected_middle = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [2, "b", "blob_1"], + [3, "c", "blob_2"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_middle, + /*predicate=*/nullptr, /*row_ranges=*/{Range(1, 2)})); + + // per-row reads resolve every row independently + const std::vector expected_rows = { + R"([[1, "a", "update1_0"]])", R"([[2, "b", "blob_1"]])", R"([[3, "c", "blob_2"]])", + R"([[4, "d", "update2_3"]])"}; + for (int32_t i = 0; i < static_cast(expected_rows.size()); i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), expected_rows[i]) + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateCompactedLayers) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + // Mirrors the layer shapes of Java's BlobUpdateTest.testReadCompactedBlobSequenceGroups + // (with a single base write, as the C++ write path assigns one sequence per commit): + // row id: 0 1 2 3 4 5 6 7 8 9 + // seq1: [b0 b1 b2 b3 b4 b5 b6 b7 b8 b9] + // seq2: [u20 P P u23 u24] . . . . . + // seq3: . . . . . [P u46 P u48 P] + // seq4: [P u61 P P P P P P P u69] + // result: u20 u61 b2 u23 u24 b5 u46 b7 u48 u69 + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array1 = MakeBlobUpdateArray(fields[2], {"u20", "PH", "PH", "u23", "u24"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array1})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + auto update_array2 = MakeBlobUpdateArray(fields[2], {"PH", "u46", "PH", "u48", "PH"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, {}, {"b0"}, {update_array2})); + SetFirstRowId(/*reset_first_row_id=*/5, commit_msgs2); + ASSERT_OK(Commit(table_path, commit_msgs2)); + + auto update_array3 = MakeBlobUpdateArray( + fields[2], {"PH", "u61", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "u69"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs3, WriteArray(table_path, {}, {"b0"}, {update_array3})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs3); + ASSERT_OK(Commit(table_path, commit_msgs3)); + + const std::vector expected_blobs = {"u20", "u61", "blob_2", "u23", "u24", + "blob_5", "u46", "blob_7", "u48", "u69"}; + auto expected_row = [&](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"" + expected_blobs[i] + + "\""; + }; + auto expected_full = PrepareBulkData(10, expected_row, fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); + + // every row resolves independently under row-range pushdown + for (int32_t i = 0; i < 10; i++) { + auto expected_single = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), + "[[" + expected_row(i) + "]]") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_single, + /*predicate=*/nullptr, /*row_ranges=*/{Range(i, i)})); + } +} + +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateWithRowRanges) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + + auto base_array = PrepareBulkData( + 10, + [](int32_t i) { + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", \"blob_" + + std::to_string(i) + "\""; + }, + fields); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {base_array})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // partial update touching rows 1 and 9 only + auto update_array = MakeBlobUpdateArray( + fields[2], {"PH", "update_1", "PH", "PH", "PH", "PH", "PH", "PH", "PH", "update_9"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + // push down row ranges hitting updated and untouched rows + auto expected_array = std::dynamic_pointer_cast( + arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ + [1, "name_1", "update_1"], + [5, "name_5", "blob_5"], + [9, "name_9", "update_9"] + ])") + .ValueOrDie()); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_array, + /*predicate=*/nullptr, + /*row_ranges=*/{Range(1, 1), Range(5, 5), Range(9, 9)})); + + // full read still resolves every row + auto expected_full = PrepareBulkData( + 10, + [](int32_t i) { + std::string blob = (i == 1 || i == 9) ? "\"update_" + std::to_string(i) + "\"" + : "\"blob_" + std::to_string(i) + "\""; + return std::to_string(i) + ", \"name_" + std::to_string(i) + "\", " + blob; + }, + fields); + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), expected_full)); +} + +/// A BLOB sequence layer whose physical file covers only a strict subrange of the full row id +/// range: the newest group is internally Gap(2), File([u2, u3]), Gap(6). On a BLOB-only table +/// the blob bunch itself carries the row-tracking fields, so the fallback reader must keep +/// _ROW_ID correct and report each row's _SEQUENCE_NUMBER from the layer that resolved it, +/// without ever exposing the internal placeholder sentinel. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateRowTrackingWithSubrangeLayer) { + arrow::FieldVector fields = {BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // old layer: rows 0-9 all real + std::vector> base_rows; + for (int32_t i = 0; i < 10; i++) { + base_rows.emplace_back("b" + std::to_string(i)); + } + auto base_array = MakeBlobUpdateArray(fields[0], base_rows); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // new layer: first_row_id=2, row_count=2, covering only rows 2-3 + auto update_array = MakeBlobUpdateArray(fields[0], {"u2", "u3"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/2, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 10); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + int64_t old_layer_seq = seq_col->Value(0); + int64_t new_layer_seq = seq_col->Value(2); + ASSERT_GT(new_layer_seq, old_layer_seq); + for (int64_t i = 0; i < 10; i++) { + ASSERT_FALSE(blob_col->IsNull(i)); + std::string expected_blob = + (i == 2 || i == 3) ? "u" + std::to_string(i) : "b" + std::to_string(i); + // the leading and trailing gaps never expose the internal placeholder sentinel + ASSERT_EQ(blob_col->GetString(i), expected_blob) << "row " << i; + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_EQ(seq_col->Value(i), (i == 2 || i == 3) ? new_layer_seq : old_layer_seq) + << "row " << i; + } +} + +/// A row that is a placeholder in every BLOB layer degrades to a null blob but keeps its +/// _ROW_ID and reports -1 as its _SEQUENCE_NUMBER. +TEST_P(BlobTableInteTest, TestDataEvolutionBlobPartialUpdateAllPlaceholderRowTracking) { + arrow::FieldVector fields = {BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + // the base layer itself holds a placeholder at row 1, so after the second layer also + // leaves it untouched, row 1 is a placeholder in every layer + auto base_array = MakeBlobUpdateArray(fields[0], {"blob_a", "PH", "blob_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, WriteArray(table_path, {}, {"b0"}, {base_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs0); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + auto update_array = MakeBlobUpdateArray(fields[0], {"PH", "PH", "update_c"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, + ScanAndReadResult(table_path, {"b0", "_ROW_ID", "_SEQUENCE_NUMBER"})); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_array = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_array); + ASSERT_EQ(struct_array->length(), 3); + auto blob_col = + std::dynamic_pointer_cast(struct_array->GetFieldByName("b0")); + auto row_id_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::RowId().Name())); + auto seq_col = std::dynamic_pointer_cast( + struct_array->GetFieldByName(SpecialFields::SequenceNumber().Name())); + ASSERT_TRUE(blob_col && row_id_col && seq_col); + + ASSERT_EQ(blob_col->GetString(0), "blob_a"); + ASSERT_EQ(blob_col->GetString(2), "update_c"); + // the all-placeholder row: null blob, row id kept, sequence number -1 + ASSERT_TRUE(blob_col->IsNull(1)); + ASSERT_EQ(row_id_col->Value(1), 1); + ASSERT_EQ(seq_col->Value(1), -1); + for (int64_t i : {static_cast(0), static_cast(2)}) { + ASSERT_EQ(row_id_col->Value(i), i); + ASSERT_GE(seq_col->Value(i), 0); + } + ASSERT_GT(seq_col->Value(2), seq_col->Value(0)); +} + +/// A normal (full-row) write never interprets blob bytes: a user value whose bytes exactly +/// equal the placeholder sentinel must be stored verbatim (not persisted as a bin_length -2 +/// entry) and survive a later placeholder fallback unchanged instead of being resolved away. +TEST_P(BlobTableInteTest, TestBlobValueEqualToPlaceholderSentinelBytes) { + arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), + arrow::field("f1", arrow::utf8()), + BlobUtils::ToArrowField("b0", /*nullable=*/true)}; + std::map options = {{Options::MANIFEST_FORMAT, "orc"}, + {Options::FILE_FORMAT, GetParam()}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(fields, /*partition_keys=*/{}, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + auto schema = arrow::schema(fields); + std::string sentinel_bytes = std::string(BlobDefs::PlaceholderSentinelView()); + + // full-row write: row 0's blob is exactly the sentinel bytes + auto struct_type = arrow::struct_(fields); + arrow::StructBuilder struct_builder( + struct_type, arrow::default_memory_pool(), + {std::make_shared(), std::make_shared(), + std::make_shared()}); + auto f0_builder = static_cast(struct_builder.field_builder(0)); + auto f1_builder = static_cast(struct_builder.field_builder(1)); + auto b0_builder = static_cast(struct_builder.field_builder(2)); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(1).ok()); + ASSERT_TRUE(f1_builder->Append("a").ok()); + ASSERT_TRUE(b0_builder->Append(sentinel_bytes.data(), sentinel_bytes.size()).ok()); + ASSERT_TRUE(struct_builder.Append().ok()); + ASSERT_TRUE(f0_builder->Append(2).ok()); + ASSERT_TRUE(f1_builder->Append("b").ok()); + ASSERT_TRUE(b0_builder->Append("normal", 6).ok()); + std::shared_ptr src_array; + ASSERT_TRUE(struct_builder.Finish(&src_array).ok()); + auto src_struct = std::dynamic_pointer_cast(src_array); + ASSERT_OK_AND_ASSIGN(auto commit_msgs0, + WriteArray(table_path, {}, schema->field_names(), {src_struct})); + ASSERT_OK(Commit(table_path, commit_msgs0)); + + // read back unchanged: the sentinel-equal bytes are a normal value + ASSERT_OK(ScanAndRead(table_path, schema->field_names(), src_struct)); + + // a partial update leaves row 0 untouched: the fallback must resolve it back to the + // sentinel-equal user bytes rather than treating the stored value as a placeholder + auto update_array = MakeBlobUpdateArray(fields[2], {"PH", "updated_b"}); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, {}, {"b0"}, {update_array})); + SetFirstRowId(/*reset_first_row_id=*/0, commit_msgs1); + ASSERT_OK(Commit(table_path, commit_msgs1)); + + ASSERT_OK_AND_ASSIGN(auto scan_read, ScanAndReadResult(table_path, schema->field_names())); + ASSERT_TRUE(scan_read.chunked_array); + auto concat_array = arrow::Concatenate(scan_read.chunked_array->chunks()).ValueOrDie(); + auto struct_result = std::dynamic_pointer_cast(concat_array); + ASSERT_TRUE(struct_result); + ASSERT_EQ(struct_result->length(), 2); + auto blob_col = + std::dynamic_pointer_cast(struct_result->GetFieldByName("b0")); + ASSERT_TRUE(blob_col); + ASSERT_FALSE(blob_col->IsNull(0)); + ASSERT_EQ(blob_col->GetString(0), sentinel_bytes); + ASSERT_EQ(blob_col->GetString(1), "updated_b"); +} + TEST_P(BlobTableInteTest, TestDataEvolutionBlobOnlyFirstCommitFailsWithoutFirstRowId) { arrow::FieldVector fields = {arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8()), BlobUtils::ToArrowField("b0")};