Skip to content

[fix](iceberg) Honor partial name mappings for legacy files #65784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Gabriel39 wants to merge 13 commits into
base: master
Choose a base branch
from
Open

[fix](iceberg) Honor partial name mappings for legacy files #65784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
d3b1a18
[fix](iceberg) Honor partial name mappings for legacy files
Gabriel39 Jul 18, 2026
c880805
fix(iceberg): preserve name mapping presence
Gabriel39 Jul 18, 2026
54ab787
fix(iceberg): preserve mapping semantics across upgrades
Gabriel39 Jul 19, 2026
adfe227
[fix](iceberg) align mixed-id nested projection semantics
Gabriel39 Jul 19, 2026
3050fa2
[fix](iceberg) preserve initial defaults across schema evolution
Gabriel39 Jul 19, 2026
b4909eb
[fix](iceberg) align pinned schemas and idless wrappers
Gabriel39 Jul 19, 2026
f1675c6
[fix](iceberg) pin name mapping and parquet id mode
Gabriel39 Jul 19, 2026
512c431
[fix](iceberg) address review feedback and FE UT
Gabriel39 Jul 20, 2026
c4f800c
[fix](iceberg) preserve file-wide schema projection
Gabriel39 Jul 20, 2026
1984bf3
[fix](iceberg) retain full schema identity for projection
Gabriel39 Jul 20, 2026
0efab7b
[fix](iceberg) gate scan semantics for rolling upgrades
Gabriel39 Jul 20, 2026
b301e6c
[fix](iceberg) align renamed field projection
Gabriel39 Jul 20, 2026
04fff46
[fix](iceberg) gate exact-name precedence
Gabriel39 Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 72 additions & 29 deletions be/src/exec/scan/access_path_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,18 @@ void inherit_schema_metadata(format::ColumnDefinition* column,
return;
}
column->name_mapping = schema_column->name_mapping;
// The presence bit is part of the mapping contract: an explicit empty mapping must remain
// authoritative after access-path pruning instead of enabling current-name fallback.
column->has_name_mapping = schema_column->has_name_mapping;
Comment thread
Gabriel39 marked this conversation as resolved.
// Initial defaults describe the logical value of fields absent from older files. Nested
// access-path pruning must retain them just like it retains rename metadata.
column->initial_default_value = schema_column->initial_default_value;
column->initial_default_value_is_base64 = schema_column->initial_default_value_is_base64;
}

const format::ColumnDefinition* find_schema_child_by_path(
const format::ColumnDefinition* schema_column, const std::string& child_path) {
const format::ColumnDefinition* schema_column, const std::string& child_path,
bool prefer_exact_name_match) {
if (schema_column == nullptr) {
return nullptr;
}
Expand All @@ -103,15 +111,31 @@ const format::ColumnDefinition* find_schema_child_by_path(
});
return child_it == schema_column->children.end() ? nullptr : &*child_it;
}
const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) {
if (to_lower(child.name) == to_lower(child_path)) {
return true;
}
if (!prefer_exact_name_match) {
const auto child_it = std::ranges::find_if(schema_column->children, [&](const auto& child) {
if (to_lower(child.name) == to_lower(child_path)) {
return true;
}
return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) {
return to_lower(alias) == to_lower(child_path);
});
});
return child_it == schema_column->children.end() ? nullptr : &*child_it;
}
// Iceberg can reuse a historical name for a newly added sibling. Current names therefore
// have precedence across the entire struct; an earlier alias must not steal that access path.
const auto exact_it = std::ranges::find_if(schema_column->children, [&](const auto& child) {
return to_lower(child.name) == to_lower(child_path);
});
if (exact_it != schema_column->children.end()) {
return &*exact_it;
}
const auto alias_it = std::ranges::find_if(schema_column->children, [&](const auto& child) {
return std::ranges::any_of(child.name_mapping, [&](const std::string& alias) {
return to_lower(alias) == to_lower(child_path);
});
});
return child_it == schema_column->children.end() ? nullptr : &*child_it;
return alias_it == schema_column->children.end() ? nullptr : &*alias_it;
}

int32_t schema_field_id(const format::ColumnDefinition* schema_column) {
Expand Down Expand Up @@ -169,7 +193,8 @@ void insert_access_path(AccessPathNode* root, const std::vector<std::string>& pa
Status build_nested_children_from_access_node(format::ColumnDefinition* column,
const DataTypePtr& type, const AccessPathNode& node,
const std::string& path,
const format::ColumnDefinition* schema_column);
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match);

// Expand a full complex-column projection into table-schema children when the table format provides
// an external/current schema. Without this, `SELECT complex_col` or `SELECT *` leaves
Expand All @@ -186,7 +211,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column,
// wrapper here: ColumnMapper and TableReader treat MAP children as [key, value].
Status build_all_nested_children_from_schema(format::ColumnDefinition* column,
const DataTypePtr& type, const std::string& path,
const format::ColumnDefinition* schema_column) {
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match) {
DORIS_CHECK(column != nullptr);

const auto nested_type = remove_nullable(type);
Expand All @@ -197,14 +223,16 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column,
const auto& struct_type = assert_cast<const DataTypeStruct&>(*nested_type);
for (size_t field_idx = 0; field_idx < struct_type.get_elements().size(); ++field_idx) {
const auto field_name = struct_type.get_element_name(field_idx);
const auto* schema_child = find_schema_child_by_path(schema_column, field_name);
const auto* schema_child =
find_schema_child_by_path(schema_column, field_name, prefer_exact_name_match);
auto* child = find_or_add_child(
column, schema_field_id_or(schema_child, cast_set<int32_t>(field_idx)),
schema_field_name_or(schema_child, field_name),
struct_type.get_element(field_idx));
inherit_schema_metadata(child, schema_child);
RETURN_IF_ERROR(build_nested_children_from_access_node(
child, child->type, project_all, path + "." + child->name, schema_child));
child, child->type, project_all, path + "." + child->name, schema_child,
prefer_exact_name_match));
}
return Status::OK();
}
Expand All @@ -217,7 +245,7 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column,
array_type.get_nested_type());
inherit_schema_metadata(child, element_schema);
return build_nested_children_from_access_node(child, child->type, project_all, path + ".*",
element_schema);
element_schema, prefer_exact_name_match);
}
case TYPE_MAP: {
const auto& map_type = assert_cast<const DataTypeMap&>(*nested_type);
Expand All @@ -231,12 +259,14 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column,
map_type.get_key_type());
inherit_schema_metadata(key_child, key_schema);
RETURN_IF_ERROR(build_nested_children_from_access_node(
key_child, key_child->type, project_all, path + ".KEYS", key_schema));
key_child, key_child->type, project_all, path + ".KEYS", key_schema,
prefer_exact_name_match));
auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value",
map_type.get_value_type());
inherit_schema_metadata(value_child, value_schema);
RETURN_IF_ERROR(build_nested_children_from_access_node(
value_child, value_child->type, project_all, path + ".VALUES", value_schema));
value_child, value_child->type, project_all, path + ".VALUES", value_schema,
prefer_exact_name_match));
return Status::OK();
}
default:
Expand All @@ -247,7 +277,8 @@ Status build_all_nested_children_from_schema(format::ColumnDefinition* column,
Status build_struct_children_from_access_node(format::ColumnDefinition* column,
const DataTypeStruct& struct_type,
const AccessPathNode& node, const std::string& path,
const format::ColumnDefinition* schema_column) {
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match) {
DORIS_CHECK(column != nullptr);
for (const auto& [child_path, child_node] : node.children) {
// Struct children are resolved by name or schema field id. We do not treat a numeric
Expand All @@ -262,7 +293,8 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column,

// Prefer the table/schema ColumnDefinition because it carries field ids and aliases.
// Fallback to the struct type name only for formats without external schema metadata.
const auto* schema_child = find_schema_child_by_path(schema_column, child_path);
const auto* schema_child =
find_schema_child_by_path(schema_column, child_path, prefer_exact_name_match);
int32_t field_id = schema_field_id(schema_child);
std::string field_name = schema_child == nullptr ? child_path : schema_child->name;
DataTypePtr field_type = schema_child == nullptr ? nullptr : schema_child->type;
Expand All @@ -288,15 +320,17 @@ Status build_struct_children_from_access_node(format::ColumnDefinition* column,
auto* child = find_or_add_child(column, field_id, field_name, field_type);
inherit_schema_metadata(child, schema_child);
RETURN_IF_ERROR(build_nested_children_from_access_node(
child, child->type, child_node, path + "." + child_path, schema_child));
child, child->type, child_node, path + "." + child_path, schema_child,
prefer_exact_name_match));
}
return Status::OK();
}

Status build_map_children_from_access_node(format::ColumnDefinition* column,
const DataTypeMap& map_type, const AccessPathNode& node,
const std::string& path,
const format::ColumnDefinition* schema_column) {
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match) {
DORIS_CHECK(column != nullptr);
AccessPathNode key_node;
AccessPathNode value_node;
Expand Down Expand Up @@ -368,33 +402,37 @@ Status build_map_children_from_access_node(format::ColumnDefinition* column,
map_type.get_key_type());
inherit_schema_metadata(key_child, key_schema);
RETURN_IF_ERROR(build_nested_children_from_access_node(key_child, key_child->type, key_node,
path + ".KEYS", key_schema));
path + ".KEYS", key_schema,
prefer_exact_name_match));
}
if (need_value) {
auto* value_child = find_or_add_child(column, schema_field_id_or(value_schema, 1), "value",
map_type.get_value_type());
inherit_schema_metadata(value_child, value_schema);
RETURN_IF_ERROR(build_nested_children_from_access_node(
value_child, value_child->type, value_node, path + ".VALUES", value_schema));
value_child, value_child->type, value_node, path + ".VALUES", value_schema,
prefer_exact_name_match));
}
return Status::OK();
}

Status build_nested_children_from_access_node(format::ColumnDefinition* column,
const DataTypePtr& type, const AccessPathNode& node,
const std::string& path,
const format::ColumnDefinition* schema_column) {
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match) {
DORIS_CHECK(column != nullptr);
if (node.project_all || node.children.empty()) {
return build_all_nested_children_from_schema(column, type, path, schema_column);
return build_all_nested_children_from_schema(column, type, path, schema_column,
prefer_exact_name_match);
}

const auto nested_type = remove_nullable(type);
switch (nested_type->get_primitive_type()) {
case TYPE_STRUCT:
return build_struct_children_from_access_node(
column, assert_cast<const DataTypeStruct&>(*nested_type), node, path,
schema_column);
column, assert_cast<const DataTypeStruct&>(*nested_type), node, path, schema_column,
prefer_exact_name_match);
case TYPE_ARRAY: {
if (node.children.size() != 1 || !node.children.contains("*")) {
return Status::NotSupported(
Expand All @@ -409,11 +447,13 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column,
array_type.get_nested_type());
inherit_schema_metadata(child, element_schema);
return build_nested_children_from_access_node(child, child->type, node.children.at("*"),
path + ".*", element_schema);
path + ".*", element_schema,
prefer_exact_name_match);
}
case TYPE_MAP:
return build_map_children_from_access_node(
column, assert_cast<const DataTypeMap&>(*nested_type), node, path, schema_column);
column, assert_cast<const DataTypeMap&>(*nested_type), node, path, schema_column,
prefer_exact_name_match);
default:
return Status::NotSupported("AccessPathParser does not support access path {} for slot {}",
path, column->name);
Expand All @@ -424,7 +464,8 @@ Status build_nested_children_from_access_node(format::ColumnDefinition* column,

Status AccessPathParser::build_nested_children(format::ColumnDefinition* column,
const std::vector<TColumnAccessPath>& access_paths,
const format::ColumnDefinition* schema_column) {
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match) {
DORIS_CHECK(column != nullptr);
if (is_scanner_materialized_virtual_column(column->name)) {
return Status::OK();
Expand Down Expand Up @@ -465,15 +506,17 @@ Status AccessPathParser::build_nested_children(format::ColumnDefinition* column,
}
// Recursively build nested children for the column based on the AccessPathNode tree.
return build_nested_children_from_access_node(column, column->type, root, column->name,
schema_column);
schema_column, prefer_exact_name_match);
}

Status AccessPathParser::build_nested_children(format::ColumnDefinition* column,
const SlotDescriptor* slot_desc,
const format::ColumnDefinition* schema_column) {
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match) {
DORIS_CHECK(column != nullptr);
DORIS_CHECK(slot_desc != nullptr);
return build_nested_children(column, slot_desc->all_access_paths(), schema_column);
return build_nested_children(column, slot_desc->all_access_paths(), schema_column,
prefer_exact_name_match);
}

} // namespace doris
6 changes: 4 additions & 2 deletions be/src/exec/scan/access_path_parser.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,13 @@ class AccessPathParser {
public:
static Status build_nested_children(format::ColumnDefinition* column,
const SlotDescriptor* slot_desc,
const format::ColumnDefinition* schema_column);
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match = true);

static Status build_nested_children(format::ColumnDefinition* column,
const std::vector<TColumnAccessPath>& access_paths,
const format::ColumnDefinition* schema_column);
const format::ColumnDefinition* schema_column,
bool prefer_exact_name_match = true);
};

} // namespace doris
7 changes: 6 additions & 1 deletion be/src/exec/scan/file_scanner_v2.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
#include "exprs/vexpr_context.h"
#include "exprs/vslot_ref.h"
#include "format/format_common.h"
#include "format/table/iceberg_scan_semantics.h"
#include "format_v2/column_mapper.h"
#include "format_v2/jni/iceberg_sys_table_reader.h"
#include "format_v2/jni/jdbc_reader.h"
Expand Down Expand Up @@ -631,6 +632,9 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_
.range = &_current_range,
.runtime_state = _state,
};
// Field 34 is the rollout boundary for root and nested exact-name precedence.
const bool prefer_exact_name_match =
!_params->__isset.history_schema_info || supports_iceberg_scan_semantics_v1(_params);

for (size_t slot_idx = 0; slot_idx < _params->required_slots.size(); ++slot_idx) {
const auto& slot_info = _params->required_slots[slot_idx];
Expand All @@ -651,7 +655,8 @@ Status FileScannerV2::_build_projected_columns(const format::TableReader& table_
// column's nested children.
RETURN_IF_ERROR(AccessPathParser::build_nested_children(
&column, it->second,
build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr));
build_context.schema_column.has_value() ? &*build_context.schema_column : nullptr,
prefer_exact_name_match));
if (is_partition_slot(slot_info, column.name)) {
column.is_partition_key = true;
_partition_slot_descs.emplace(
Expand Down
Loading
Loading