From 83f2db77912f941733c9d864670ba60d458f71d8 Mon Sep 17 00:00:00 2001 From: JunRuiLee Date: Thu, 23 Jul 2026 10:59:18 +0800 Subject: [PATCH] feat(table): build a catalog-free Table from a portable TableDescriptor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add TableDescriptor (path + full TableSchema + options) — a slim, versioned, cross-language JSON wire form for rebuilding a read-only table without a catalog lookup — plus Table::from_descriptor_json and the C FFI paimon_table_from_descriptor. Includes a golden byte-identical to the Java TableDescriptorSerializer output for cross-language validation. --- .licenserc.yaml | 1 + bindings/c/src/table.rs | 102 ++++- bindings/c/src/tests.rs | 52 +++ crates/paimon/src/table/descriptor.rs | 379 ++++++++++++++++++ .../table/goldens/table_descriptor_v1.json | 34 ++ crates/paimon/src/table/mod.rs | 2 + 6 files changed, 567 insertions(+), 3 deletions(-) create mode 100644 crates/paimon/src/table/descriptor.rs create mode 100644 crates/paimon/src/table/goldens/table_descriptor_v1.json diff --git a/.licenserc.yaml b/.licenserc.yaml index c3cac61b..a1cee6c2 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -27,6 +27,7 @@ header: - ".gitattributes" - ".github/PULL_REQUEST_TEMPLATE.md" - "crates/paimon/tests/**/*.json" + - "crates/paimon/src/table/goldens/*.json" - "crates/paimon/testdata/**" - "third-party-licenses/openssl-1.1.1.LICENSE" - "**/go.sum" diff --git a/bindings/c/src/table.rs b/bindings/c/src/table.rs index 05f82e31..f2a77054 100644 --- a/bindings/c/src/table.rs +++ b/bindings/c/src/table.rs @@ -17,6 +17,7 @@ use std::collections::HashMap; use std::ffi::c_void; +use std::ptr; use arrow_array::ffi::{FFI_ArrowArray, FFI_ArrowSchema}; use arrow_array::{Array, StructArray}; @@ -27,8 +28,9 @@ use paimon::Plan; use crate::error::{check_non_null, paimon_error, validate_cstr, PaimonErrorCode}; use crate::result::{ - paimon_result_new_read, paimon_result_next_batch, paimon_result_plan, paimon_result_predicate, - paimon_result_read_builder, paimon_result_record_batch_reader, paimon_result_table_scan, + paimon_result_get_table, paimon_result_new_read, paimon_result_next_batch, paimon_result_plan, + paimon_result_predicate, paimon_result_read_builder, paimon_result_record_batch_reader, + paimon_result_table_scan, }; use crate::runtime; use crate::types::*; @@ -61,12 +63,96 @@ unsafe fn box_table_read_state(state: TableReadState) -> *mut paimon_table_read /// Free a paimon_table. /// /// # Safety -/// Only call with a table returned from `paimon_catalog_get_table`. +/// Only call with a table returned from `paimon_catalog_get_table` or +/// `paimon_table_from_descriptor`. #[no_mangle] pub unsafe extern "C" fn paimon_table_free(table: *mut paimon_table) { free_table_wrapper(table, |t| t.inner); } +/// Build a catalog-free `paimon_table` (intended for scan/read) from an +/// FE-provided `TableDescriptor` JSON, without any catalog lookup. +/// +/// `descriptor_json` is a NUL-terminated UTF-8 JSON string (see the Rust +/// `paimon::table::TableDescriptor`). `options` is an array of `options_len` +/// storage/runtime `paimon_option` values (credentials, endpoints), or null +/// when `options_len` is 0; they configure `FileIO` only and never change table +/// semantics. The returned table is intended for scan/read. +/// +/// On success `table` is non-null and `error` is null; free it with +/// `paimon_table_free`. On failure `table` is null and `error` is non-null. +/// +/// # Safety +/// `descriptor_json` must be a valid C string. `options` must point to +/// `options_len` valid `paimon_option` values, or be null when `options_len` is 0. +#[no_mangle] +pub unsafe extern "C" fn paimon_table_from_descriptor( + descriptor_json: *const std::ffi::c_char, + options: *const paimon_option, + options_len: usize, +) -> paimon_result_get_table { + let json = match validate_cstr(descriptor_json, "descriptor_json") { + Ok(s) => s, + Err(e) => { + return paimon_result_get_table { + table: ptr::null_mut(), + error: e, + } + } + }; + + if options.is_null() && options_len > 0 { + return paimon_result_get_table { + table: ptr::null_mut(), + error: paimon_error::new( + PaimonErrorCode::InvalidInput, + "null options pointer with non-zero length".to_string(), + ), + }; + } + let mut opts: HashMap = HashMap::new(); + if options_len > 0 { + let options_slice = std::slice::from_raw_parts(options, options_len); + for opt in options_slice { + let key = match validate_cstr(opt.key, "option key") { + Ok(s) => s, + Err(e) => { + return paimon_result_get_table { + table: ptr::null_mut(), + error: e, + } + } + }; + let value = match validate_cstr(opt.value, "option value") { + Ok(s) => s, + Err(e) => { + return paimon_result_get_table { + table: ptr::null_mut(), + error: e, + } + } + }; + opts.insert(key, value); + } + } + + match Table::from_descriptor_json(&json, opts) { + Ok(table) => { + let wrapper = Box::new(paimon_table { + inner: Box::into_raw(Box::new(table)) as *mut c_void, + }); + paimon_result_get_table { + table: Box::into_raw(wrapper), + error: ptr::null_mut(), + } + } + Err(e) => paimon_result_get_table { + table: ptr::null_mut(), + error: paimon_error::from_paimon(e), + }, + } +} + /// Time-travel selector option names, in the core's resolution priority order. const TIME_TRAVEL_SELECTORS: [&str; 4] = [ "scan.timestamp-millis", @@ -1787,6 +1873,16 @@ const _: unsafe extern "C" fn( const _: unsafe extern "C" fn(*const u8, usize) -> paimon_result_plan = paimon_plan_from_split_bytes; +// Descriptor table constructor ABI signature guard. Pins the catalog-free table +// constructor so an accidental signature change fails to compile rather than +// silently breaking header consumers. To add behavior, introduce a new symbol +// instead of changing this one. +const _: unsafe extern "C" fn( + *const std::ffi::c_char, + *const paimon_option, + usize, +) -> paimon_result_get_table = paimon_table_from_descriptor; + #[cfg(test)] mod tests { use super::*; diff --git a/bindings/c/src/tests.rs b/bindings/c/src/tests.rs index 58fb27ab..e805f33a 100644 --- a/bindings/c/src/tests.rs +++ b/bindings/c/src/tests.rs @@ -2474,3 +2474,55 @@ fn plan_from_split_bytes_rejects_garbage() { assert!(!r.error.is_null()); unsafe { paimon_error_free(r.error) }; } + +// A minimal but valid v1 TableDescriptor JSON (scalar schema; the descriptor +// parser's complex-type coverage lives in the paimon crate's unit tests). +const DESCRIPTOR_V1_JSON: &str = r#"{ + "version": 1, + "path": "file:///tmp/paimon-c-test/db.db/t", + "database": "db", + "name": "t", + "tableSchema": { + "version": 3, + "id": 0, + "fields": [ { "id": 0, "name": "id", "type": "INT NOT NULL" } ], + "highestFieldId": 0, + "partitionKeys": [], + "primaryKeys": ["id"], + "options": {}, + "timeMillis": 1 + } +}"#; + +#[test] +fn table_from_descriptor_builds_table() { + let json = CString::new(DESCRIPTOR_V1_JSON).unwrap(); + unsafe { + let result = paimon_table_from_descriptor(json.as_ptr(), ptr::null(), 0); + assert!(result.error.is_null(), "from_descriptor should not error"); + assert!(!result.table.is_null(), "table handle must be non-null"); + paimon_table_free(result.table); + } +} + +#[test] +fn table_from_descriptor_rejects_unknown_version() { + let json = + CString::new(DESCRIPTOR_V1_JSON.replace("\"version\": 1", "\"version\": 2")).unwrap(); + unsafe { + let result = paimon_table_from_descriptor(json.as_ptr(), ptr::null(), 0); + assert!(!result.error.is_null(), "unknown version must error"); + assert!(result.table.is_null()); + paimon_error_free(result.error); + } +} + +#[test] +fn table_from_descriptor_rejects_null_json() { + unsafe { + let result = paimon_table_from_descriptor(ptr::null(), ptr::null(), 0); + assert!(!result.error.is_null(), "null json must error"); + assert!(result.table.is_null()); + paimon_error_free(result.error); + } +} diff --git a/crates/paimon/src/table/descriptor.rs b/crates/paimon/src/table/descriptor.rs new file mode 100644 index 00000000..b0597b90 --- /dev/null +++ b/crates/paimon/src/table/descriptor.rs @@ -0,0 +1,379 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Portable, versioned table descriptor for catalog-free table construction. +//! +//! A [`TableDescriptor`] is the cross-language wire form an engine (e.g. Doris +//! BE) uses to rebuild a catalog-free [`Table`](crate::table::Table) from +//! FE-resolved metadata — table path + full [`TableSchema`] + options — without +//! opening a catalog. It mirrors the Java `org.apache.paimon.table.TableDescriptor` +//! DTO and is deliberately slim: it carries only what a reader needs. +//! +//! The descriptor's own [`version`](TableDescriptor::version) is distinct from +//! the nested `tableSchema`'s schema-format version. + +use std::collections::HashMap; + +use serde::Deserialize; + +use crate::catalog::{Identifier, DEFAULT_MAIN_BRANCH, UNKNOWN_DATABASE}; +use crate::error::Error; +use crate::io::FileIO; +use crate::spec::{TableSchema, PATH_OPTION}; +use crate::table::Table; +use crate::Result; + +/// The only descriptor contract version this build understands. +pub(crate) const TABLE_DESCRIPTOR_VERSION: i32 = 1; + +/// A portable, versioned description of a Paimon table, sufficient to build a +/// catalog-free [`Table`](crate::table::Table) without any catalog lookup. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TableDescriptor { + /// Descriptor contract version. Distinct from `table_schema.version`. + pub version: i32, + /// Table root path (location). Authoritative for `FileIO`/`SchemaManager`. + pub path: String, + /// Full table schema — identical to an on-disk `schema/schema-N` file. + /// + /// Deserialized by the existing [`TableSchema`] serde, so it faithfully + /// carries every field Rust models. Fields Rust does not model yet (e.g. + /// Java `DataField.defaultValue`) are tolerated but ignored; see the design + /// doc's v2 notes. + pub table_schema: TableSchema, + /// Optional dynamic/read option overlay. Reserved in v1: it is NOT merged + /// into the table schema (that would silently alter table semantics). + #[serde(default)] + pub options: HashMap, + /// Optional database name (used for the [`Identifier`](crate::catalog::Identifier) + /// and diagnostics). + #[serde(default)] + pub database: Option, + /// Optional table name (used for the identifier and diagnostics). + #[serde(default, rename = "name")] + pub table_name: Option, + /// Optional branch. v1 supports only the main branch; any other value is + /// rejected up front rather than silently reading main. + #[serde(default)] + pub branch: Option, +} + +impl TableDescriptor { + /// Parse and validate a [`TableDescriptor`] from its JSON wire form. + pub fn from_json(json: &str) -> Result { + let descriptor: TableDescriptor = + serde_json::from_str(json).map_err(|e| Error::DataInvalid { + message: "Failed to parse TableDescriptor".to_string(), + source: Some(Box::new(e)), + })?; + descriptor.validate()?; + Ok(descriptor) + } + + /// Reject descriptors this build cannot honor: unknown contract versions, + /// and (v1) any non-main branch. Runs before any filesystem access. + fn validate(&self) -> Result<()> { + if self.version != TABLE_DESCRIPTOR_VERSION { + return Err(Error::Unsupported { + message: format!( + "TableDescriptor version {} is not supported (expected {})", + self.version, TABLE_DESCRIPTOR_VERSION + ), + }); + } + if let Some(branch) = self.branch.as_deref() { + if branch != DEFAULT_MAIN_BRANCH { + return Err(Error::Unsupported { + message: format!( + "TableDescriptor v1 only supports the main branch; got branch={branch:?}" + ), + }); + } + } + Ok(()) + } +} + +impl Table { + /// Build a catalog-free [`Table`] from an FE-provided [`TableDescriptor`] + /// JSON. No catalog lookup is performed. + /// + /// `storage_options` are storage/runtime options (credentials, endpoints) + /// supplied by the embedding engine; they configure [`FileIO`] only and never + /// alter table semantics. Table semantics come solely from the descriptor's + /// `table_schema`. + /// + /// The result is intended for scan/read use: it carries no REST environment + /// (`rest_env = None`), so REST snapshot-commit is unavailable. + pub fn from_descriptor_json( + json: &str, + storage_options: HashMap, + ) -> Result { + let TableDescriptor { + path, + table_schema, + database, + table_name, + .. + } = TableDescriptor::from_json(json)?; + + // Identifier: require both db and name, or neither. Exactly one present + // signals an incomplete producer payload and is rejected rather than + // silently synthesized. + let identifier = match (database, table_name) { + (Some(db), Some(name)) => { + let id = Identifier::new(db, name); + id.validate()?; + id + } + (None, None) => Identifier::new(UNKNOWN_DATABASE, "table"), + _ => { + return Err(Error::DataInvalid { + message: "TableDescriptor must set both `database` and `name`, or neither" + .to_string(), + source: None, + }) + } + }; + + // Path rule: the top-level `path` is authoritative. Inject/overwrite it + // into the schema options so `Table.location` and the schema agree + // (matters especially for format tables). + let table_schema = table_schema + .copy_with_options(HashMap::from([(PATH_OPTION.to_string(), path.clone())])); + + // Plain FileIO from the table path + engine-supplied storage options; no + // REST token machinery (the slim descriptor carries none). + let file_io = FileIO::from_path(&path)? + .with_props(storage_options) + .build()?; + + Ok(Table::new(file_io, identifier, path, table_schema, None)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A minimal but realistic v1 descriptor. The nested `tableSchema` includes + /// a complex (VECTOR) type so the test exercises Paimon's custom type serde, + /// not just scalar strings. + fn valid_descriptor_json() -> String { + r#"{ + "version": 1, + "path": "file:///tmp/paimon/mydb.db/t", + "database": "mydb", + "name": "t", + "tableSchema": { + "version": 3, + "id": 5, + "fields": [ + { "id": 0, "name": "id", "type": "INT NOT NULL" }, + { "id": 1, "name": "embedding", "type": { "type": "VECTOR", "element": "FLOAT", "length": 2 } } + ], + "highestFieldId": 1, + "partitionKeys": [], + "primaryKeys": ["id"], + "options": { "bucket": "1", "merge-engine": "deduplicate" }, + "timeMillis": 1784359714317 + }, + "options": {} + }"# + .to_string() + } + + #[test] + fn parses_valid_v1_descriptor() { + let d = TableDescriptor::from_json(&valid_descriptor_json()).expect("should parse"); + assert_eq!(d.version, 1); + assert_eq!(d.path, "file:///tmp/paimon/mydb.db/t"); + assert_eq!(d.database.as_deref(), Some("mydb")); + assert_eq!(d.table_name.as_deref(), Some("t")); + assert_eq!(d.table_schema.id(), 5); + assert_eq!(d.table_schema.fields().len(), 2); + assert_eq!(d.table_schema.primary_keys(), &["id".to_string()]); + } + + #[test] + fn rejects_unknown_version() { + let json = valid_descriptor_json().replace("\"version\": 1", "\"version\": 2"); + let err = TableDescriptor::from_json(&json).expect_err("v2 must be rejected"); + assert!( + matches!(err, Error::Unsupported { .. }), + "expected Unsupported, got {err:?}" + ); + } + + #[test] + fn rejects_non_main_branch() { + let json = valid_descriptor_json().replace( + "\"database\": \"mydb\",", + "\"database\": \"mydb\", \"branch\": \"b1\",", + ); + let err = TableDescriptor::from_json(&json).expect_err("non-main branch must be rejected"); + assert!( + matches!(err, Error::Unsupported { .. }), + "expected Unsupported, got {err:?}" + ); + } + + #[test] + fn accepts_explicit_main_branch() { + let json = valid_descriptor_json().replace( + "\"database\": \"mydb\",", + "\"database\": \"mydb\", \"branch\": \"main\",", + ); + let d = TableDescriptor::from_json(&json).expect("main branch is fine"); + assert_eq!(d.branch.as_deref(), Some("main")); + } + + #[test] + fn missing_required_field_is_data_invalid() { + // Drop the required top-level `path`. + let json = + valid_descriptor_json().replace("\"path\": \"file:///tmp/paimon/mydb.db/t\",", ""); + let err = TableDescriptor::from_json(&json).expect_err("missing path must fail"); + assert!( + matches!(err, Error::DataInvalid { .. }), + "expected DataInvalid, got {err:?}" + ); + } + + #[test] + fn malformed_json_is_data_invalid() { + let err = TableDescriptor::from_json("{ not json").expect_err("bad json must fail"); + assert!( + matches!(err, Error::DataInvalid { .. }), + "expected DataInvalid, got {err:?}" + ); + } + + #[test] + fn options_default_to_empty_when_absent() { + let json = valid_descriptor_json().replace(",\n \"options\": {}", ""); + let d = TableDescriptor::from_json(&json).expect("options is optional"); + assert!(d.options.is_empty()); + } + + #[test] + fn builds_catalog_free_table_from_descriptor() { + let table = Table::from_descriptor_json(&valid_descriptor_json(), HashMap::new()) + .expect("should build table"); + assert_eq!(table.location(), "file:///tmp/paimon/mydb.db/t"); + assert_eq!(table.schema().id(), 5); + assert_eq!(table.identifier().database(), "mydb"); + assert_eq!(table.identifier().object(), "t"); + assert_eq!( + table + .schema() + .options() + .get(PATH_OPTION) + .map(String::as_str), + Some("file:///tmp/paimon/mydb.db/t"), + "top-level path must be injected into schema options" + ); + } + + #[test] + fn synthesizes_identifier_when_names_absent() { + let json = valid_descriptor_json() + .replace("\"database\": \"mydb\",", "") + .replace("\"name\": \"t\",", ""); + let table = Table::from_descriptor_json(&json, HashMap::new()).expect("names are optional"); + assert_eq!(table.identifier().database(), UNKNOWN_DATABASE); + } + + #[test] + fn rejects_partial_identifier() { + // Exactly one of database/name present is an incomplete payload. + let json = valid_descriptor_json().replace("\"name\": \"t\",", ""); + let err = Table::from_descriptor_json(&json, HashMap::new()) + .expect_err("partial identifier must be rejected"); + assert!( + matches!(err, Error::DataInvalid { .. }), + "expected DataInvalid, got {err:?}" + ); + } + + #[test] + fn rejects_empty_branch() { + let json = valid_descriptor_json().replace( + "\"database\": \"mydb\",", + "\"database\": \"mydb\", \"branch\": \"\",", + ); + let err = TableDescriptor::from_json(&json).expect_err("empty branch must be rejected"); + assert!( + matches!(err, Error::Unsupported { .. }), + "expected Unsupported, got {err:?}" + ); + } + + #[test] + fn descriptor_options_do_not_override_schema_or_path() { + // v1: descriptor.options must NOT leak into table semantics. + let json = valid_descriptor_json().replace( + "\"options\": {}", + "\"options\": { \"path\": \"file:///bad\", \"bucket\": \"999\" }", + ); + let table = Table::from_descriptor_json(&json, HashMap::new()).expect("should build"); + // Top-level `path` wins over descriptor.options["path"]. + assert_eq!(table.location(), "file:///tmp/paimon/mydb.db/t"); + assert_eq!( + table + .schema() + .options() + .get(PATH_OPTION) + .map(String::as_str), + Some("file:///tmp/paimon/mydb.db/t") + ); + // Table semantics come from tableSchema.options, not descriptor.options. + assert_eq!( + table.schema().options().get("bucket").map(String::as_str), + Some("1") + ); + } + + #[test] + fn parses_java_produced_golden_v1() { + // Cross-language contract: this fixture is byte-identical to the golden produced by the + // Java `TableDescriptorSerializer` (apache/paimon `TableDescriptorGoldenTest`, + // `table-descriptor-v1.json`). If the Java side changes the wire form, this test fails — + // keep the two goldens in sync. + let json = include_str!("goldens/table_descriptor_v1.json"); + let table = + Table::from_descriptor_json(json, HashMap::new()).expect("Java golden must parse"); + + assert_eq!(table.location(), "file:///tmp/warehouse/mydb.db/t"); + assert_eq!(table.identifier().database(), "mydb"); + assert_eq!(table.identifier().object(), "t"); + + let schema = table.schema(); + assert_eq!(schema.id(), 1); + assert_eq!(schema.highest_field_id(), 2); + assert_eq!(schema.fields().len(), 3); + assert_eq!(schema.primary_keys(), &["id".to_string()]); + // Third field is an ARRAY encoded as a nested object by Java; confirm the custom + // type serde actually reconstructed the Array type, not just the field name. + assert_eq!(schema.fields()[2].name(), "tags"); + assert!(matches!( + schema.fields()[2].data_type(), + crate::spec::DataType::Array(_) + )); + } +} diff --git a/crates/paimon/src/table/goldens/table_descriptor_v1.json b/crates/paimon/src/table/goldens/table_descriptor_v1.json new file mode 100644 index 00000000..52134f20 --- /dev/null +++ b/crates/paimon/src/table/goldens/table_descriptor_v1.json @@ -0,0 +1,34 @@ +{ + "version" : 1, + "path" : "file:///tmp/warehouse/mydb.db/t", + "tableSchema" : { + "version" : 3, + "id" : 1, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT NOT NULL" + }, { + "id" : 1, + "name" : "name", + "type" : "VARCHAR(100)" + }, { + "id" : 2, + "name" : "tags", + "type" : { + "type" : "ARRAY", + "element" : "INT" + } + } ], + "highestFieldId" : 2, + "partitionKeys" : [ ], + "primaryKeys" : [ "id" ], + "options" : { + "bucket" : "1" + }, + "comment" : "canonical", + "timeMillis" : 1700000000000 + }, + "database" : "mydb", + "name" : "t" +} \ No newline at end of file diff --git a/crates/paimon/src/table/mod.rs b/crates/paimon/src/table/mod.rs index 5c34918d..8bac8801 100644 --- a/crates/paimon/src/table/mod.rs +++ b/crates/paimon/src/table/mod.rs @@ -38,6 +38,7 @@ pub mod data_evolution_writer; mod data_file_reader; mod data_file_writer; mod dedicated_format_file_writer; +pub(crate) mod descriptor; mod format_read_builder; mod format_table_read; mod format_table_scan; @@ -94,6 +95,7 @@ pub use btree_global_index_build_builder::BTreeGlobalIndexBuildBuilder; pub use commit_message::CommitMessage; pub use cow_writer::{CopyOnWriteMergeWriter, FileInfo}; pub use data_evolution_writer::{DataEvolutionDeleteWriter, DataEvolutionWriter}; +pub use descriptor::TableDescriptor; #[cfg(feature = "fulltext")] pub use full_text_search_builder::FullTextSearchBuilder; use futures::stream::BoxStream;