diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d7d00002..3dd22f37a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,9 +66,17 @@ option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) option(PAIMON_ENABLE_TANTIVY "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) +option(PAIMON_ENABLE_REST "Whether to enable the rest catalog (requires libcurl)" OFF) if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() +if(PAIMON_ENABLE_REST) + add_definitions(-DPAIMON_ENABLE_REST) +endif() +# libcurl backs the HTTP client shared by the S3 file system and the rest catalog. +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + find_package(CURL REQUIRED) +endif() if(PAIMON_ENABLE_AVRO) add_definitions(-DPAIMON_ENABLE_AVRO) endif() @@ -487,9 +495,6 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake" config_summary_message() -if(PAIMON_ENABLE_S3) - find_package(CURL REQUIRED) -endif() add_subdirectory(src/paimon) add_subdirectory(src/paimon/fs/local) add_subdirectory(src/paimon/fs/jindo) diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 339d6b163..872fc0f90 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -130,6 +130,7 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_ENABLE_REST=ON" "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) diff --git a/docs/source/building.rst b/docs/source/building.rst index 66f932414..87e39ae16 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -145,6 +145,7 @@ boolean flags to ``cmake``. * ``-DPAIMON_ENABLE_AVRO=ON``: Apache Avro libraries and Paimon integration * ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for Lumina vector index, lumina is only supported on gcc9 or higher. +* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog (``metastore=rest``), requires the libcurl development package. Third-party dependency source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h new file mode 100644 index 000000000..49b61b8ac --- /dev/null +++ b/include/paimon/catalog_options.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "paimon/visibility.h" + +namespace paimon { + +/// Catalog-level configuration option keys; table-level keys live in `Options`. +struct PAIMON_EXPORT CatalogOptions { + /// "metastore" - Metastore of the paimon catalog. + /// Supported values are "filesystem" (default) and "rest". + static const char METASTORE[]; + + /// "uri" - Server url of the REST catalog. Only used when METASTORE is "rest". + static const char URI[]; + + /// "token" - Token of the "bear" token provider of the REST catalog. + static const char TOKEN[]; + + /// "token.provider" - Authentication provider of the REST catalog. + /// Currently only "bear" is supported ("bear" is the protocol's historical + /// spelling of "bearer", do not "fix" it). + static const char TOKEN_PROVIDER[]; + + /// "table-default." - Prefix of the catalog options that provide table option + /// defaults: "table-default.=" applies "=" to a created + /// table when the caller left "" unset. + static const char TABLE_DEFAULT_OPTION_PREFIX[]; +}; + +} // namespace paimon diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h index c5c796150..def91f390 100644 --- a/include/paimon/utils/special_field_ids.h +++ b/include/paimon/utils/special_field_ids.h @@ -39,6 +39,10 @@ class SpecialFieldIds { /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1 static const int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1; + + /// The lowest field ID reserved for system fields; IDs at or above this bound are + /// excluded when computing the highest field ID of a schema. Value: INT32_MAX / 2 + static const int32_t SYSTEM_FIELD_ID_START = std::numeric_limits::max() / 2; }; } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index 409c58a50..31c22b283 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -13,6 +13,7 @@ # limitations under the License. set(PAIMON_COMMON_SRCS + common/catalog_options.cpp common/compression/block_compression_factory.cpp common/compression/block_compressor.cpp common/compression/block_decompressor.cpp @@ -176,12 +177,18 @@ set(PAIMON_COMMON_SRCS common/utils/roaring_bitmap64.cpp common/utils/row_range_index.cpp common/utils/status.cpp - common/utils/string_utils.cpp) + common/utils/string_utils.cpp + common/utils/url_utils.cpp) +# The shared HTTP client is used by both the object store file systems and the +# rest catalog. +set(PAIMON_CURL_LINK_LIBS) +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + list(APPEND PAIMON_COMMON_SRCS common/utils/http_client.cpp) + set(PAIMON_CURL_LINK_LIBS CURL::libcurl) +endif() if(PAIMON_ENABLE_S3) - list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp - common/fs/object_store_file_system.cpp) - set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl) + list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp) endif() set(PAIMON_CORE_SRCS @@ -224,6 +231,7 @@ set(PAIMON_CORE_SRCS core/casting/timestamp_to_timestamp_cast_executor.cpp core/casting/casting_utils.cpp core/catalog/catalog.cpp + core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp core/core_options.cpp @@ -403,6 +411,18 @@ set(PAIMON_CORE_SRCS core/utils/special_field_ids.cpp core/utils/tag_manager.cpp) +if(PAIMON_ENABLE_REST) + list(APPEND + PAIMON_CORE_SRCS + rest/rest_http_client.cpp + rest/resource_paths.cpp + rest/rest_api.cpp + rest/rest_auth.cpp + rest/rest_catalog.cpp + rest/rest_messages.cpp + rest/rest_util.cpp) +endif() + add_paimon_lib(paimon SOURCES ${PAIMON_COMMON_SRCS} @@ -416,7 +436,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} STATIC_LINK_LIBS arrow tbb @@ -426,7 +446,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) @@ -434,6 +454,13 @@ add_subdirectory(common/file_index) add_subdirectory(common/global_index) if(PAIMON_BUILD_TESTS) + # The shared HTTP client is compiled only for the components that need libcurl, + # so its test follows the same gate. + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS) + if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS common/utils/http_client_test.cpp) + endif() + add_paimon_test(memory_test SOURCES common/memory/memory_pool_test.cpp @@ -594,6 +621,8 @@ if(PAIMON_BUILD_TESTS) common/utils/status_test.cpp common/utils/stream_utils_test.cpp common/utils/string_utils_test.cpp + common/utils/url_utils_test.cpp + ${PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS} common/utils/range_test.cpp common/utils/uuid_test.cpp common/utils/decimal_utils_test.cpp @@ -870,4 +899,18 @@ if(PAIMON_BUILD_TESTS) EXTRA_INCLUDES ${JINDOSDK_INCLUDE_DIR}) + if(PAIMON_ENABLE_REST) + add_paimon_test(rest_test + SOURCES + rest/rest_http_client_test.cpp + rest/mock_rest_server.cpp + rest/rest_catalog_test.cpp + rest/rest_messages_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + ${TEST_STATIC_LINK_LIBS} + ${GTEST_LINK_TOOLCHAIN}) + endif() + endif() diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp new file mode 100644 index 000000000..6e89e80a8 --- /dev/null +++ b/src/paimon/common/catalog_options.cpp @@ -0,0 +1,27 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/catalog_options.h" + +namespace paimon { + +const char CatalogOptions::METASTORE[] = "metastore"; +const char CatalogOptions::URI[] = "uri"; +const char CatalogOptions::TOKEN[] = "token"; +const char CatalogOptions::TOKEN_PROVIDER[] = "token.provider"; +const char CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX[] = "table-default."; + +} // namespace paimon diff --git a/src/paimon/common/fs/http_client.cpp b/src/paimon/common/utils/http_client.cpp similarity index 90% rename from src/paimon/common/fs/http_client.cpp rename to src/paimon/common/utils/http_client.cpp index afd816ce8..e9a39acf1 100644 --- a/src/paimon/common/fs/http_client.cpp +++ b/src/paimon/common/utils/http_client.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include @@ -44,11 +44,6 @@ class CurlGlobalGuard { } }; -std::shared_ptr GetCurlGlobalGuard() { - static auto guard = std::make_shared(); - return guard; -} - void TrimHttpWhitespace(std::string* value) { constexpr char kHttpWhitespace[] = " \t\r\n"; size_t begin = value->find_first_not_of(kHttpWhitespace); @@ -92,16 +87,7 @@ size_t WriteCallback(char* data, size_t size, size_t count, void* user_data) { size_t HeaderCallback(char* data, size_t size, size_t count, void* user_data) { auto* context = static_cast(user_data); size_t bytes = size * count; - std::string line(data, bytes); - size_t colon = line.find(':'); - if (colon != std::string::npos) { - std::string name = line.substr(0, colon); - TrimHttpWhitespace(&name); - name = StringUtils::ToLowerCase(name); - std::string value = line.substr(colon + 1); - TrimHttpWhitespace(&value); - context->response.headers[name] = std::move(value); - } + ParseHttpHeaderLine(data, bytes, &context->response.headers); return bytes; } @@ -116,9 +102,31 @@ bool IsRetryable(CURLcode code, int64_t status_code) { } // namespace +std::shared_ptr EnsureCurlGlobalInit() { + static auto guard = std::make_shared(); + return guard; +} + +void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers) { + std::string line(data, size); + size_t colon = line.find(':'); + if (colon == std::string::npos) { + return; + } + std::string name = line.substr(0, colon); + TrimHttpWhitespace(&name); + if (name.empty()) { + return; + } + name = StringUtils::ToLowerCase(name); + std::string value = line.substr(colon + 1); + TrimHttpWhitespace(&value); + (*headers)[name] = std::move(value); +} + class CurlHttpClient::Impl { public: - Impl() : guard_(GetCurlGlobalGuard()) {} + Impl() : guard_(EnsureCurlGlobalInit()) {} ~Impl() { for (CURL* handle : handles_) { @@ -143,7 +151,7 @@ class CurlHttpClient::Impl { } private: - std::shared_ptr guard_; + std::shared_ptr guard_; mutable std::mutex mutex_; mutable std::vector handles_; }; diff --git a/src/paimon/common/fs/http_client.h b/src/paimon/common/utils/http_client.h similarity index 80% rename from src/paimon/common/fs/http_client.h rename to src/paimon/common/utils/http_client.h index 586972223..0a11e39cd 100644 --- a/src/paimon/common/fs/http_client.h +++ b/src/paimon/common/utils/http_client.h @@ -31,6 +31,14 @@ enum class HttpMethod { HEAD, GET }; using HttpHeaders = std::map; using HttpBodyConsumer = std::function; +/// Ensures libcurl's global state is initialized; the returned guard keeps it alive. +PAIMON_EXPORT std::shared_ptr EnsureCurlGlobalInit(); + +/// Parses one raw HTTP header line into `headers`, lower-casing the name and trimming +/// HTTP whitespace around the name and value; lines without a ':' or with an empty +/// name are ignored. +PAIMON_EXPORT void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers); + struct HttpRequest { HttpMethod method = HttpMethod::GET; std::string url; diff --git a/src/paimon/common/utils/http_client_test.cpp b/src/paimon/common/utils/http_client_test.cpp new file mode 100644 index 000000000..5bf3ce232 --- /dev/null +++ b/src/paimon/common/utils/http_client_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/http_client.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(HttpClientUtilTest, ParseHttpHeaderLine) { + HttpHeaders headers; + const std::string line = "Content-Type: application/json\r\n"; + ParseHttpHeaderLine(line.data(), line.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "application/json"}}), headers); + + const std::string overwrite = "content-type: text/PLAIN \r\n"; + ParseHttpHeaderLine(overwrite.data(), overwrite.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "text/PLAIN"}}), headers); + + // Lines without a colon (like the status line) and empty names are ignored. + const std::string status_line = "HTTP/1.1 200 OK\r\n"; + ParseHttpHeaderLine(status_line.data(), status_line.size(), &headers); + const std::string empty_name = ": value\r\n"; + ParseHttpHeaderLine(empty_name.data(), empty_name.size(), &headers); + ASSERT_EQ(1, headers.size()); + + const std::string empty_value = "x-empty:\r\n"; + ParseHttpHeaderLine(empty_value.data(), empty_value.size(), &headers); + ASSERT_EQ("", headers.at("x-empty")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/utils/url_utils.cpp b/src/paimon/common/utils/url_utils.cpp new file mode 100644 index 000000000..33cb00836 --- /dev/null +++ b/src/paimon/common/utils/url_utils.cpp @@ -0,0 +1,134 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include + +namespace paimon { + +namespace { + +bool IsAlphaNumeric(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +/// The characters `EncodeString` keeps unencoded. +bool IsFormUnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '.' || c == '-' || c == '*' || c == '_'; +} + +/// The unreserved characters of RFC 3986. +bool IsRfc3986UnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '-' || c == '.' || c == '_' || c == '~'; +} + +void AppendPercentEncoded(char c, std::string* out) { + char buf[4]; + std::snprintf(buf, sizeof(buf), "%%%02X", static_cast(c)); + out->append(buf); +} + +int32_t HexValue(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +} // namespace + +std::string UrlUtils::EncodeString(const std::string& input) { + std::string encoded; + encoded.reserve(input.size()); + for (char c : input) { + if (IsFormUnreservedChar(c)) { + encoded.push_back(c); + } else if (c == ' ') { + encoded.push_back('+'); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +std::string UrlUtils::DecodeString(const std::string& input) { + std::string decoded; + decoded.reserve(input.size()); + for (size_t i = 0; i < input.size(); i++) { + char c = input[i]; + if (c == '+') { + decoded.push_back(' '); + } else if (c == '%' && i + 2 < input.size()) { + int32_t high = HexValue(input[i + 1]); + int32_t low = HexValue(input[i + 2]); + if (high >= 0 && low >= 0) { + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } else { + decoded.push_back(c); + } + } + return decoded; +} + +std::string UrlUtils::PercentEncode(std::string_view value, bool preserve_slash) { + std::string encoded; + encoded.reserve(value.size()); + for (char c : value) { + if (IsRfc3986UnreservedChar(c) || (preserve_slash && c == '/')) { + encoded.push_back(c); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +Result UrlUtils::PercentDecode(std::string_view value) { + std::string decoded; + decoded.reserve(value.size()); + for (size_t i = 0; i < value.size(); i++) { + char c = value[i]; + if (c == '%') { + if (i + 2 >= value.size()) { + return Status::IOError("invalid percent encoding in URL component"); + } + int32_t high = HexValue(value[i + 1]); + int32_t low = HexValue(value[i + 2]); + if (high < 0 || low < 0) { + return Status::IOError("invalid percent encoding in URL component"); + } + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } + return decoded; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils.h b/src/paimon/common/utils/url_utils.h new file mode 100644 index 000000000..481acdf1e --- /dev/null +++ b/src/paimon/common/utils/url_utils.h @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// URL encoding and decoding shared by the REST catalog and the object store file +/// systems. +class PAIMON_EXPORT UrlUtils { + public: + UrlUtils() = delete; + ~UrlUtils() = delete; + + /// URL-encode a string in the `application/x-www-form-urlencoded` flavor (UTF-8): + /// alphanumeric characters and ".", "-", "*", "_" are kept, a space becomes "+", all + /// other bytes become percent-encoded "%XX". This is the flavor the REST catalog + /// server expects; note it escapes "~", unlike RFC 3986. + static std::string EncodeString(const std::string& input); + + /// Decodes `EncodeString` output ('+' back to space, "%XX" to the byte); malformed + /// escape sequences are kept as-is instead of failing. + static std::string DecodeString(const std::string& input); + + /// Percent-encode a URL component with RFC 3986 rules: alphanumeric characters and + /// "-", ".", "_", "~" are kept, every other byte (including a space) becomes "%XX". + /// With `preserve_slash`, '/' is kept as-is so an object key keeps its path shape. + static std::string PercentEncode(std::string_view value, bool preserve_slash = false); + + /// Strict inverse of `PercentEncode`: "%XX" becomes the byte, '+' is kept as-is, and + /// a '%' not followed by two hex digits is an error. + static Result PercentDecode(std::string_view value); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils_test.cpp b/src/paimon/common/utils/url_utils_test.cpp new file mode 100644 index 000000000..e85860e45 --- /dev/null +++ b/src/paimon/common/utils/url_utils_test.cpp @@ -0,0 +1,81 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include "gtest/gtest.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(UrlUtilsTest, EncodeString) { + ASSERT_EQ("abcDEF012.-*_", UrlUtils::EncodeString("abcDEF012.-*_")); + ASSERT_EQ("a+b", UrlUtils::EncodeString("a b")); + ASSERT_EQ("a%2Fb", UrlUtils::EncodeString("a/b")); + ASSERT_EQ("a%3Db%26c", UrlUtils::EncodeString("a=b&c")); + // '~' is the character distinguishing the two encoding flavors: EncodeString + // (form-urlencoded) escapes it while PercentEncode (RFC 3986) keeps it. + ASSERT_EQ("%7E", UrlUtils::EncodeString("~")); + ASSERT_EQ("a%2Bb", UrlUtils::EncodeString("a+b")); + ASSERT_EQ("%E4%B8%AD", UrlUtils::EncodeString("中")); +} + +TEST(UrlUtilsTest, DecodeString) { + ASSERT_EQ("a/b", UrlUtils::DecodeString("a%2Fb")); + ASSERT_EQ("a b", UrlUtils::DecodeString("a+b")); + ASSERT_EQ("中", UrlUtils::DecodeString("%E4%B8%AD")); + ASSERT_EQ("a/", UrlUtils::DecodeString("a%2F")); + // Malformed escapes are kept as-is. + ASSERT_EQ("a%", UrlUtils::DecodeString("a%")); + ASSERT_EQ("%2", UrlUtils::DecodeString("%2")); + ASSERT_EQ("%ZZ", UrlUtils::DecodeString("%ZZ")); + ASSERT_EQ("100%", UrlUtils::DecodeString("100%")); +} + +TEST(UrlUtilsTest, PercentEncode) { + ASSERT_EQ("abcDEF012-._~", UrlUtils::PercentEncode("abcDEF012-._~")); + ASSERT_EQ("a%20b", UrlUtils::PercentEncode("a b")); + ASSERT_EQ("a%2Ab", UrlUtils::PercentEncode("a*b")); + ASSERT_EQ("a%2Fb", UrlUtils::PercentEncode("a/b")); + ASSERT_EQ("a/b%3Dc", UrlUtils::PercentEncode("a/b=c", /*preserve_slash=*/true)); + ASSERT_EQ("%E4%B8%AD", UrlUtils::PercentEncode("中")); +} + +TEST(UrlUtilsTest, PercentDecode) { + ASSERT_OK_AND_ASSIGN(std::string decoded, UrlUtils::PercentDecode("a%2Fb%20c")); + ASSERT_EQ("a/b c", decoded); + // '+' is form encoding only; percent decoding keeps it. + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("a+b")); + ASSERT_EQ("a+b", decoded); + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("%e4%b8%ad")); + ASSERT_EQ("中", decoded); + // Malformed escapes are an error. + ASSERT_NOK(UrlUtils::PercentDecode("a%").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%2").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%ZZ").status()); + ASSERT_NOK(UrlUtils::PercentDecode("100%").status()); +} + +TEST(UrlUtilsTest, RoundTrip) { + const std::string input = "db 1/table$branch_b1=中%"; + ASSERT_EQ(input, UrlUtils::DecodeString(UrlUtils::EncodeString(input))); + ASSERT_OK_AND_ASSIGN(std::string decoded, + UrlUtils::PercentDecode(UrlUtils::PercentEncode(input))); + ASSERT_EQ(input, decoded); +} + +} // namespace paimon::test diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index 80f93cb29..5b704d866 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -18,8 +18,13 @@ #include +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" +#ifdef PAIMON_ENABLE_REST +#include "paimon/rest/rest_catalog.h" +#endif namespace paimon { @@ -31,6 +36,22 @@ const char Catalog::DB_LOCATION_PROP[] = "location"; Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { + std::string metastore = "filesystem"; + auto metastore_iter = options.find(CatalogOptions::METASTORE); + if (metastore_iter != options.end()) { + metastore = StringUtils::ToLowerCase(metastore_iter->second); + } + if (metastore == "rest") { +#ifdef PAIMON_ENABLE_REST + return RestCatalog::Create(root_path, options, file_system); +#else + return Status::NotImplemented( + "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); +#endif + } + if (metastore != "filesystem") { + return Status::Invalid("unsupported metastore: ", metastore); + } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); return std::make_unique(core_options.GetFileSystem(), root_path, options); } diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp new file mode 100644 index 000000000..99352115e --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/catalog_utils.h" + +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/result.h" + +namespace paimon { + +bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { + return db_name == Catalog::SYSTEM_DATABASE_NAME; +} + +Status CatalogUtils::CheckNotSystemTable(const Identifier& identifier, const std::string& action) { + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return Status::Invalid( + fmt::format("Cannot '{}' for system table '{}', please use data table.", action, + identifier.ToString())); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + return Status::Invalid( + fmt::format("Cannot '{}' for system table '{}', please use data table.", action, + identifier.ToString())); + } + return Status::OK(); +} + +Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + if (branch) { + return Status::Invalid(fmt::format( + "Cannot '{}' for branch table '{}', please modify the table with the default branch.", + action, identifier.ToString())); + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h new file mode 100644 index 000000000..f2602e1ec --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.h @@ -0,0 +1,43 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/status.h" + +namespace paimon { + +/// Checks shared by the catalog implementations. +class CatalogUtils { + public: + CatalogUtils() = delete; + ~CatalogUtils() = delete; + + /// Returns whether `db_name` is the reserved system database "sys". + static bool IsSystemDatabase(const std::string& db_name); + + /// Fails when `identifier` denotes a system table or a table inside the system + /// database. + static Status CheckNotSystemTable(const Identifier& identifier, const std::string& action); + + /// Fails when `identifier` carries a branch. + static Status CheckNotBranch(const Identifier& identifier, const std::string& action); +}; + +} // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index ec142736f..ca6d3d886 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -29,6 +29,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/system/global_system_tables.h" @@ -134,12 +135,7 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* const std::vector& primary_keys, const std::map& options, bool ignore_if_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid( - fmt::format("Cannot create table for system table {}, please use data table.", - identifier.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "create table")); PAIMON_ASSIGN_OR_RAISE(bool db_exist, DatabaseExists(identifier.GetDatabaseName())); if (!db_exist) { return Status::Invalid( @@ -197,7 +193,7 @@ const std::map& FileSystemCatalog::GetOptions() const } bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { - return db_name == SYSTEM_DATABASE_NAME; + return CatalogUtils::IsSystemDatabase(db_name); } Result FileSystemCatalog::IsSpecifiedSystemTable(const Identifier& identifier) { @@ -425,10 +421,7 @@ Status FileSystemCatalog::DropTableImpl(const Identifier& identifier, } Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid(fmt::format("Cannot drop system table {}.", identifier.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "drop table")); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); if (!exist) { @@ -488,12 +481,8 @@ Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_from_system_table, IsSystemTable(from_table)); - PAIMON_ASSIGN_OR_RAISE(bool is_to_system_table, IsSystemTable(to_table)); - if (is_from_system_table || is_to_system_table) { - return Status::Invalid(fmt::format("Cannot rename system table {} or {}.", - from_table.ToString(), to_table.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "rename table")); if (from_table.GetDatabaseName() != to_table.GetDatabaseName()) { return Status::Invalid( @@ -521,24 +510,6 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi return Status::OK(); } -namespace { -SnapshotInfo::CommitKind ConvertCommitKind(Snapshot::CommitKind internal) { - if (internal == Snapshot::CommitKind::Append()) { - return SnapshotInfo::CommitKind::APPEND; - } - if (internal == Snapshot::CommitKind::Compact()) { - return SnapshotInfo::CommitKind::COMPACT; - } - if (internal == Snapshot::CommitKind::Overwrite()) { - return SnapshotInfo::CommitKind::OVERWRITE; - } - if (internal == Snapshot::CommitKind::Analyze()) { - return SnapshotInfo::CommitKind::ANALYZE; - } - return SnapshotInfo::CommitKind::UNKNOWN; -} -} // namespace - Result> FileSystemCatalog::ListSnapshots( const Identifier& identifier, const std::string& branch) const { PAIMON_ASSIGN_OR_RAISE(bool exists, TableExists(identifier)); @@ -553,20 +524,9 @@ Result> FileSystemCatalog::ListSnapshots( std::vector result; result.reserve(snapshots.size()); - for (const auto& snap : snapshots) { - SnapshotInfo info; - info.snapshot_id = snap.Id(); - info.schema_id = snap.SchemaId(); - info.commit_user = snap.CommitUser(); - info.commit_kind = ConvertCommitKind(snap.GetCommitKind()); - info.time_millis = snap.TimeMillis(); - info.total_record_count = snap.TotalRecordCount(); - info.delta_record_count = snap.DeltaRecordCount(); - info.watermark = snap.Watermark(); - result.push_back(std::move(info)); + result.push_back(snap.ToSnapshotInfo()); } - return result; } diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index a48ff6ce1..a0eaa63f7 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -114,7 +114,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(Identifier("db1", "ta$ble"), &schema, {"f1"}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&schema); } } @@ -206,11 +206,12 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(options_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), + "Cannot 'drop table' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(options_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); } TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { @@ -277,11 +278,12 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(audit_log_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), + "Cannot 'drop table' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(audit_log_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); } TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { @@ -420,11 +422,12 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(snapshots_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'create table' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), + "Cannot 'drop table' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(snapshots_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); } TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { @@ -785,10 +788,10 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_FALSE(exist); /// Test 4: Drop system table. - ASSERT_NOK_WITH_MSG( - catalog.DropTable(Identifier("test_db", "tbl$system"), - /*ignore_if_not_exists=*/false), - "Cannot drop system table Identifier{database='test_db', table='tbl$system'}."); + ASSERT_NOK_WITH_MSG(catalog.DropTable(Identifier("test_db", "tbl$system"), + /*ignore_if_not_exists=*/false), + "Cannot 'drop table' for system table 'Identifier{database='test_db', " + "table='tbl$system'}', please use data table."); ArrowSchemaRelease(&schema); } @@ -854,7 +857,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("test_db", "tbl$system"), Identifier("test_db", "new_system_tbl"), /*ignore_if_not_exists=*/false), - "Cannot rename system table"); + "Cannot 'rename table' for system table"); ArrowSchemaRelease(&schema1); ArrowSchemaRelease(&schema2); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index b991d59f1..1fe2f1bfb 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -36,6 +36,7 @@ #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/defs.h" #include "paimon/status.h" +#include "paimon/utils/special_field_ids.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" #include "rapidjson/rapidjson.h" @@ -234,6 +235,68 @@ Result> TableSchema::CreateFromJson(const std::stri table_schema.options_, table_schema.comment_, table_schema.time_millis_); } +namespace { + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id); + +Status CollectTypeFieldIds(const rapidjson::Value& type, std::set* seen_ids, + int32_t* max_id) { + if (!type.IsObject()) { + return Status::OK(); + } + // ROW type carries nested "fields", ARRAY/MULTISET carry "element" and MAP carries + // "key"/"value". + if (type.HasMember("fields") && type["fields"].IsArray()) { + PAIMON_RETURN_NOT_OK(CollectFieldIds(type["fields"], seen_ids, max_id)); + } + if (type.HasMember("element")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["element"], seen_ids, max_id)); + } + if (type.HasMember("key")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["key"], seen_ids, max_id)); + } + if (type.HasMember("value")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["value"], seen_ids, max_id)); + } + return Status::OK(); +} + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id) { + for (const auto& field : fields.GetArray()) { + if (!field.IsObject()) { + return Status::Invalid("Broken schema, a field must be an object."); + } + if (!field.HasMember("id") || !field["id"].IsInt()) { + return Status::Invalid("Broken schema, a field misses an integer id."); + } + int32_t id = field["id"].GetInt(); + if (!seen_ids->insert(id).second) { + return Status::Invalid(fmt::format("Broken schema, field id {} is duplicated.", id)); + } + if (id < SpecialFieldIds::SYSTEM_FIELD_ID_START) { + *max_id = std::max(*max_id, id); + } + if (field.HasMember("type")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(field["type"], seen_ids, max_id)); + } + } + return Status::OK(); +} + +} // namespace + +Result TableSchema::ComputeHighestFieldId(const rapidjson::Value& fields) { + if (!fields.IsArray()) { + return Status::Invalid("Broken schema, 'fields' must be an array."); + } + int32_t max_id = -1; + std::set seen_ids; + PAIMON_RETURN_NOT_OK(CollectFieldIds(fields, &seen_ids, &max_id)); + return max_id; +} + Result> TableSchema::InitSchema( int64_t schema_id, const std::vector& fields, int32_t highest_field_id, const std::vector& partition_keys, const std::vector& primary_keys, diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index abbb54113..df9cb6fee 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -51,6 +51,14 @@ class TableSchema : public DataSchema, public Jsonizable { static Result> CreateFromJson(const std::string& json_str); + /// Computes the highest non-system field id (ids at or above + /// `SpecialFieldIds::SYSTEM_FIELD_ID_START` are excluded) across all nesting + /// levels of a "fields" JSON array, or -1 when there is none. A non-object field, + /// a duplicated, missing or non-integer field id fails the computation. Used when + /// the schema source (e.g. a rest catalog server) does not report the + /// "highestFieldId" itself. + static Result ComputeHighestFieldId(const rapidjson::Value& fields); + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) override; diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 4f486fa19..63e302c53 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -27,6 +27,8 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/special_field_ids.h" +#include "rapidjson/document.h" namespace paimon::test { @@ -148,6 +150,45 @@ TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) { ASSERT_EQ(variant_type, FieldType::VARIANT); } +TEST_F(TableSchemaTest, TestComputeHighestFieldId) { + auto compute = [](const std::string& fields_json) -> Result { + rapidjson::Document doc; + doc.Parse(fields_json.c_str()); + EXPECT_FALSE(doc.HasParseError()) << fields_json; + return TableSchema::ComputeHighestFieldId(doc); + }; + // ids inside nested rows are included + ASSERT_OK_AND_ASSIGN(int32_t highest, compute(R"([ + {"id": 0, "name": "f0", "type": "INT"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 5, "name": "inner", "type": "INT"}]}} + ])")); + ASSERT_EQ(5, highest); + // system field ids are reserved and excluded from the highest field id + ASSERT_OK_AND_ASSIGN(highest, + compute(R"([{"id": 3, "name": "f0", "type": "INT"}, + {"id": )" + + std::to_string(SpecialFieldIds::SEQUENCE_NUMBER) + + R"(, "name": "_SEQUENCE_NUMBER", "type": "BIGINT"}])")); + ASSERT_EQ(3, highest); + // no fields at all yields -1 + ASSERT_OK_AND_ASSIGN(highest, compute("[]")); + ASSERT_EQ(-1, highest); + // broken schemas fail instead of being silently skipped + ASSERT_NOK_WITH_MSG(compute(R"([{"id": 0, "name": "a", "type": "INT"}, + {"id": 0, "name": "b", "type": "INT"}])") + .status(), + "duplicated"); + ASSERT_NOK_WITH_MSG(compute(R"([{"name": "a", "type": "INT"}])").status(), "integer id"); + ASSERT_NOK_WITH_MSG(compute(R"([{"id": "0", "name": "a", "type": "INT"}])").status(), + "integer id"); + ASSERT_NOK_WITH_MSG(compute("[1]").status(), "must be an object"); + rapidjson::Document not_an_array; + not_an_array.Parse("{}"); + ASSERT_NOK_WITH_MSG(TableSchema::ComputeHighestFieldId(not_an_array).status(), + "must be an array"); +} + TEST_F(TableSchemaTest, TestInvalidCreate) { // partial fields have field id arrow::FieldVector fields = MakeArrowField(3); diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index 8e50a6464..f27846a12 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -313,4 +313,27 @@ Result Snapshot::FromPath(const std::shared_ptr& fs, return snapshot; } +SnapshotInfo Snapshot::ToSnapshotInfo() const { + SnapshotInfo info; + info.snapshot_id = Id(); + info.schema_id = SchemaId(); + info.commit_user = CommitUser(); + if (commit_kind_ == CommitKind::Append()) { + info.commit_kind = SnapshotInfo::CommitKind::APPEND; + } else if (commit_kind_ == CommitKind::Compact()) { + info.commit_kind = SnapshotInfo::CommitKind::COMPACT; + } else if (commit_kind_ == CommitKind::Overwrite()) { + info.commit_kind = SnapshotInfo::CommitKind::OVERWRITE; + } else if (commit_kind_ == CommitKind::Analyze()) { + info.commit_kind = SnapshotInfo::CommitKind::ANALYZE; + } else { + info.commit_kind = SnapshotInfo::CommitKind::UNKNOWN; + } + info.time_millis = TimeMillis(); + info.total_record_count = TotalRecordCount(); + info.delta_record_count = DeltaRecordCount(); + info.watermark = Watermark(); + return info; +} + } // namespace paimon diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h index 2ef09e377..6520bf3ca 100644 --- a/src/paimon/core/snapshot.h +++ b/src/paimon/core/snapshot.h @@ -25,6 +25,7 @@ #include "paimon/common/utils/jsonizable.h" #include "paimon/result.h" +#include "paimon/snapshot/snapshot_info.h" #include "paimon/type_fwd.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" @@ -129,6 +130,8 @@ class Snapshot : public Jsonizable { bool operator==(const Snapshot& other) const; bool TEST_Equal(const Snapshot& other) const; + SnapshotInfo ToSnapshotInfo() const; + public: static constexpr int64_t FIRST_SNAPSHOT_ID = 1; static constexpr int32_t TABLE_STORE_02_VERSION = 1; diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index a6388e1cb..27ac93ee4 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -327,6 +327,39 @@ TEST_F(SnapshotTest, TestSnapshotInfoCommitKindToString) { ASSERT_EQ("UNKNOWN", SnapshotInfo::CommitKindToString(SnapshotInfo::CommitKind::UNKNOWN)); } +TEST_F(SnapshotTest, TestToSnapshotInfo) { + auto make_snapshot = [](const Snapshot::CommitKind& kind) { + return Snapshot( + /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"bml", + /*base_manifest_list_size=*/10, + /*delta_manifest_list=*/"dml", /*delta_manifest_list_size=*/20, + /*changelog_manifest_list=*/std::nullopt, /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user1", /*commit_identifier=*/20, + kind, /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, + /*changelog_record_count=*/std::nullopt, /*watermark=*/50, + /*statistics=*/std::nullopt, /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); + }; + SnapshotInfo info = make_snapshot(Snapshot::CommitKind::Append()).ToSnapshotInfo(); + ASSERT_EQ(10, info.snapshot_id); + ASSERT_EQ(15, info.schema_id); + ASSERT_EQ("user1", info.commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(1234, info.time_millis); + ASSERT_EQ(35, info.total_record_count.value()); + ASSERT_EQ(40, info.delta_record_count.value()); + ASSERT_EQ(50, info.watermark.value()); + // every commit kind maps to its SnapshotInfo counterpart, anything else to UNKNOWN + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, + make_snapshot(Snapshot::CommitKind::Compact()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::OVERWRITE, + make_snapshot(Snapshot::CommitKind::Overwrite()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::ANALYZE, + make_snapshot(Snapshot::CommitKind::Analyze()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::UNKNOWN, + make_snapshot(Snapshot::CommitKind::Unknown()).ToSnapshotInfo().commit_kind); +} + TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { // Test with changelog_manifest_list set to a non-null value { diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp index c3f9d376a..948e2c6e8 100644 --- a/src/paimon/fs/s3/s3_file_system.cpp +++ b/src/paimon/fs/s3/s3_file_system.cpp @@ -42,52 +42,22 @@ #include #include "fmt/format.h" -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" #include "paimon/executor.h" namespace paimon::s3 { namespace { -Result PercentEncode(std::string_view value, bool preserve_slash) { - if (value.size() > std::numeric_limits::max()) { - return Status::IOError("S3 URL component is too large to encode"); - } - char* encoded = curl_easy_escape(nullptr, value.data(), static_cast(value.size())); - if (encoded == nullptr) { - return Status::IOError("failed to URL encode S3 component"); - } - ScopeGuard free_encoded([encoded] { curl_free(encoded); }); - std::string result(encoded); - if (preserve_slash) { - result = StringUtils::Replace(result, "%2F", "/"); - } - return result; -} - Result PercentDecode(std::string_view value, const std::string& field) { - for (size_t position = 0; position < value.size(); ++position) { - if (value[position] == '%' && - (position + 2 >= value.size() || - !std::isxdigit(static_cast(value[position + 1])) || - !std::isxdigit(static_cast(value[position + 2])))) { - return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); - } - } - if (value.size() > std::numeric_limits::max()) { - return Status::IOError(fmt::format("S3 {} is too large to URL decode", field)); + Result decoded = UrlUtils::PercentDecode(value); + if (!decoded.ok()) { + return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); } - int decoded_size = 0; - char* decoded = - curl_easy_unescape(nullptr, value.data(), static_cast(value.size()), &decoded_size); - if (decoded == nullptr) { - return Status::IOError(fmt::format("failed to URL decode S3 {}", field)); - } - ScopeGuard free_decoded([decoded] { curl_free(decoded); }); - std::string result(decoded, decoded_size); - return result; + return decoded; } Result ParseNonNegativeInt64(const std::string& value, const std::string& field) { @@ -704,13 +674,10 @@ class S3ObjectStoreClient : public ObjectStoreClient, int32_t max_keys) const override { std::string query = "list-type=2&delimiter=%2F&encoding-type=url"; if (!path.key.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_prefix, PercentEncode(path.key, false)); - query += "&prefix=" + encoded_prefix; + query += "&prefix=" + UrlUtils::PercentEncode(path.key); } if (!continuation_token.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_token, - PercentEncode(continuation_token, false)); - query += "&continuation-token=" + encoded_token; + query += "&continuation-token=" + UrlUtils::PercentEncode(continuation_token); } if (max_keys > 0) { query += "&max-keys=" + std::to_string(max_keys); @@ -847,13 +814,11 @@ class S3ObjectStoreClient : public ObjectStoreClient, : endpoint_.scheme != "http" || IsIpAddressAuthority(endpoint_.authority) || !IsVirtualHostableS3Bucket(object.bucket, true)); if (use_path_style) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_bucket, PercentEncode(object.bucket, false)); - request_path += encoded_bucket + "/"; + request_path += UrlUtils::PercentEncode(object.bucket) + "/"; } else { authority = object.bucket + "." + authority; } - PAIMON_ASSIGN_OR_RAISE(std::string encoded_key, PercentEncode(object.key, true)); - request_path += encoded_key; + request_path += UrlUtils::PercentEncode(object.key, /*preserve_slash=*/true); if (!query.empty()) { request_path += "?" + query; } diff --git a/src/paimon/fs/s3/s3_file_system.h b/src/paimon/fs/s3/s3_file_system.h index 66c796f89..bca948f5e 100644 --- a/src/paimon/fs/s3/s3_file_system.h +++ b/src/paimon/fs/s3/s3_file_system.h @@ -20,8 +20,8 @@ #include #include -#include "paimon/common/fs/http_client.h" #include "paimon/common/fs/object_store_file_system.h" +#include "paimon/common/utils/http_client.h" namespace paimon::s3 { diff --git a/src/paimon/rest/mock_rest_server.cpp b/src/paimon/rest/mock_rest_server.cpp new file mode 100644 index 000000000..a793de0ef --- /dev/null +++ b/src/paimon/rest/mock_rest_server.cpp @@ -0,0 +1,215 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/mock_rest_server.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { + +bool ReceiveAll(int32_t fd, std::string* buffer, size_t min_size) { + char chunk[4096]; + while (buffer->size() < min_size) { + ssize_t received = ::recv(fd, chunk, sizeof(chunk), 0); + if (received <= 0) { + return false; + } + buffer->append(chunk, static_cast(received)); + } + return true; +} + +std::map ParseQuery(const std::string& query) { + std::map params; + for (const std::string& pair : StringUtils::Split(query, "&", /*ignore_empty=*/true)) { + size_t eq = pair.find('='); + if (eq == std::string::npos) { + params[UrlUtils::DecodeString(pair)] = ""; + } else { + params[UrlUtils::DecodeString(pair.substr(0, eq))] = + UrlUtils::DecodeString(pair.substr(eq + 1)); + } + } + return params; +} + +} // namespace + +MockRestServer::MockRestServer(Handler handler, int32_t listen_fd, int32_t port) + : handler_(std::move(handler)), listen_fd_(listen_fd), port_(port) { + accept_thread_ = std::thread([this] { AcceptLoop(); }); +} + +Result> MockRestServer::Start(Handler handler) { + int32_t listen_fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (listen_fd < 0) { + return Status::IOError("mock rest server: failed to create socket: ", std::strerror(errno)); + } + int32_t reuse = 1; + ::setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + struct sockaddr_in address; + std::memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; + if (::bind(listen_fd, reinterpret_cast(&address), sizeof(address)) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to bind: ", std::strerror(errno)); + } + if (::listen(listen_fd, 16) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to listen: ", std::strerror(errno)); + } + socklen_t address_len = sizeof(address); + if (::getsockname(listen_fd, reinterpret_cast(&address), &address_len) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to get port: ", std::strerror(errno)); + } + int32_t port = ntohs(address.sin_port); + return std::unique_ptr(new MockRestServer(std::move(handler), listen_fd, port)); +} + +MockRestServer::~MockRestServer() { + Stop(); +} + +void MockRestServer::Stop() { + if (stopped_.exchange(true)) { + return; + } + ::shutdown(listen_fd_, SHUT_RDWR); + ::close(listen_fd_); + if (accept_thread_.joinable()) { + accept_thread_.join(); + } +} + +std::string MockRestServer::GetBaseUri() const { + return fmt::format("http://127.0.0.1:{}", port_); +} + +void MockRestServer::AcceptLoop() { + while (!stopped_.load()) { + int32_t connection_fd = ::accept(listen_fd_, nullptr, nullptr); + if (connection_fd < 0) { + if (stopped_.load()) { + return; + } + continue; + } + // The connection is handled on the accept thread; the socket timeouts keep a + // wedged peer from blocking `Stop()` indefinitely. + struct timeval timeout; + timeout.tv_sec = 30; + timeout.tv_usec = 0; + ::setsockopt(connection_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + ::setsockopt(connection_fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + HandleConnection(connection_fd); + ::close(connection_fd); + } +} + +void MockRestServer::HandleConnection(int32_t connection_fd) { + std::string buffer; + size_t header_end; + while ((header_end = buffer.find("\r\n\r\n")) == std::string::npos) { + if (!ReceiveAll(connection_fd, &buffer, buffer.size() + 1)) { + return; + } + } + + Request request; + std::string header_part = buffer.substr(0, header_end); + std::vector lines = StringUtils::Split(header_part, "\r\n", + /*ignore_empty=*/true); + if (lines.empty()) { + return; + } + std::vector request_line = StringUtils::Split(lines[0], " ", + /*ignore_empty=*/true); + if (request_line.size() < 2) { + return; + } + request.method = request_line[0]; + std::string target = request_line[1]; + size_t question = target.find('?'); + if (question == std::string::npos) { + request.path = UrlUtils::DecodeString(target); + } else { + request.path = UrlUtils::DecodeString(target.substr(0, question)); + request.query_params = ParseQuery(target.substr(question + 1)); + } + size_t content_length = 0; + for (size_t i = 1; i < lines.size(); i++) { + size_t colon = lines[i].find(':'); + if (colon == std::string::npos) { + continue; + } + std::string name = lines[i].substr(0, colon); + std::string value = lines[i].substr(colon + 1); + StringUtils::Trim(&name); + StringUtils::Trim(&value); + request.headers[StringUtils::ToLowerCase(name)] = value; + } + auto length_iter = request.headers.find("content-length"); + if (length_iter != request.headers.end()) { + content_length = + static_cast(std::strtoul(length_iter->second.c_str(), nullptr, 10)); + } + size_t body_begin = header_end + 4; + if (!ReceiveAll(connection_fd, &buffer, body_begin + content_length)) { + return; + } + request.body = buffer.substr(body_begin, content_length); + + Response response = handler_(request); + if (response.close_without_response) { + return; + } + std::string extra_headers; + for (const auto& [name, value] : response.headers) { + extra_headers += fmt::format("{}: {}\r\n", name, value); + } + std::string payload = fmt::format( + "HTTP/1.1 {} MOCK\r\nContent-Type: {}\r\nContent-Length: {}\r\n{}Connection: " + "close\r\n\r\n{}", + response.code, response.content_type, response.body.size() + response.missing_body_bytes, + extra_headers, response.body); + size_t sent = 0; + while (sent < payload.size()) { + ssize_t written = ::send(connection_fd, payload.data() + sent, payload.size() - sent, 0); + if (written <= 0) { + return; + } + sent += static_cast(written); + } +} + +} // namespace paimon diff --git a/src/paimon/rest/mock_rest_server.h b/src/paimon/rest/mock_rest_server.h new file mode 100644 index 000000000..5fd09f108 --- /dev/null +++ b/src/paimon/rest/mock_rest_server.h @@ -0,0 +1,92 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// A minimal blocking HTTP/1.1 server for REST catalog unit tests. It listens on a +/// random port of 127.0.0.1, parses one request per connection and answers it with the +/// response returned by the registered handler. Test only, not production code. +class MockRestServer { + public: + struct Request { + std::string method; + /// Url-decoded path, e.g. "/v1/config". + std::string path; + /// Url-decoded query parameters. + std::map query_params; + /// Headers with lower-cased names. + std::map headers; + std::string body; + }; + + struct Response { + int32_t code = 200; + std::string body; + std::string content_type = "application/json"; + /// Extra response headers, e.g. "Retry-After". + std::map headers; + /// Close the connection without sending any response; the client observes + /// CURLE_GOT_NOTHING, a transport error that is never retried. + bool close_without_response = false; + /// Advertise this many bytes beyond the actual body in `Content-Length` and + /// close the connection without sending them; the client observes the response + /// being cut off mid-body (CURLE_PARTIAL_FILE), a retriable transport error. + size_t missing_body_bytes = 0; + }; + + using Handler = std::function; + + /// Starts the server with `handler` invoked for every request (on the server + /// thread). + static Result> Start(Handler handler); + + ~MockRestServer(); + + void Stop(); + + int32_t GetPort() const { + return port_; + } + + /// "http://127.0.0.1:" + std::string GetBaseUri() const; + + private: + MockRestServer(Handler handler, int32_t listen_fd, int32_t port); + + void AcceptLoop(); + void HandleConnection(int32_t connection_fd); + + Handler handler_; + int32_t listen_fd_ = -1; + int32_t port_ = 0; + std::atomic stopped_{false}; + std::thread accept_thread_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.cpp b/src/paimon/rest/resource_paths.cpp new file mode 100644 index 000000000..484482378 --- /dev/null +++ b/src/paimon/rest/resource_paths.cpp @@ -0,0 +1,64 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/resource_paths.h" + +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { +constexpr const char kV1[] = "/v1"; +} // namespace + +ResourcePaths::ResourcePaths(const std::string& prefix) { + base_ = kV1; + if (!prefix.empty()) { + base_ += "/" + UrlUtils::EncodeString(prefix); + } +} + +std::string ResourcePaths::Config() { + return std::string(kV1) + "/config"; +} + +std::string ResourcePaths::Databases() const { + return base_ + "/databases"; +} + +std::string ResourcePaths::Database(const std::string& database_name) const { + return Databases() + "/" + UrlUtils::EncodeString(database_name); +} + +std::string ResourcePaths::Tables(const std::string& database_name) const { + return Database(database_name) + "/tables"; +} + +std::string ResourcePaths::Table(const std::string& database_name, + const std::string& table_name) const { + return Tables(database_name) + "/" + UrlUtils::EncodeString(table_name); +} + +std::string ResourcePaths::RenameTable() const { + return base_ + "/tables/rename"; +} + +std::string ResourcePaths::Snapshots(const std::string& database_name, + const std::string& table_name) const { + return Table(database_name, table_name) + "/snapshots"; +} + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.h b/src/paimon/rest/resource_paths.h new file mode 100644 index 000000000..ea36bf480 --- /dev/null +++ b/src/paimon/rest/resource_paths.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Builds resource paths of the REST catalog server. All path segments are +/// url-encoded; `prefix` (usually pushed down by the server through `/v1/config`) may +/// be empty, in which case it is skipped. +class ResourcePaths { + public: + explicit ResourcePaths(const std::string& prefix); + + /// "/v1/config", the only path that does not carry the prefix. + static std::string Config(); + + std::string Databases() const; + std::string Database(const std::string& database_name) const; + std::string Tables(const std::string& database_name) const; + std::string Table(const std::string& database_name, const std::string& table_name) const; + std::string RenameTable() const; + std::string Snapshots(const std::string& database_name, const std::string& table_name) const; + + private: + /// "/v1" or "/v1/{encoded prefix}". + std::string base_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp new file mode 100644 index 000000000..727f26d26 --- /dev/null +++ b/src/paimon/rest/rest_api.cpp @@ -0,0 +1,294 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_api.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/url_utils.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { +// The `PAIMON_ASSIGN_OR_RAISE` macro cannot take a declaration type containing a comma. +using StringMap = std::map; + +// Parses a response body without echoing the body (or the parse failure carrying it) +// into the error message: a successful response may contain credentials (e.g. a token +// response), so only the request path is reported. +template +Status ParseResponseBody(const std::string& body, const std::string& path, ResponseT* entity) { + if (!RapidJsonUtil::FromJsonString(body, entity).ok()) { + return Status::Invalid( + fmt::format("failed to deserialize the response of {} from the rest server", path)); + } + return Status::OK(); +} +} // namespace + +RestApi::RestApi(std::unique_ptr client, + std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths) + : client_(std::move(client)), + auth_provider_(std::move(auth_provider)), + base_headers_(base_headers), + options_(options), + resource_paths_(paths) {} + +Result> RestApi::Create(const std::map& options, + const std::string& warehouse, bool config_required, + const RestHttpClient::Config& http_config) { + auto uri_iter = options.find(CatalogOptions::URI); + if (uri_iter == options.end() || uri_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::URI)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr client, + RestHttpClient::Create(uri_iter->second, http_config)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr auth_provider, + AuthProvider::Create(options)); + + std::map merged_options = options; + std::map base_headers = + RestUtil::ExtractPrefixMap(options, kHeaderOptionPrefix); + if (config_required) { + std::map query_params; + if (!warehouse.empty()) { + query_params[kQueryParamWarehouse] = warehouse; + } + RestAuthParameter auth_parameter; + auth_parameter.method = "GET"; + auth_parameter.resource_path = ResourcePaths::Config(); + for (const auto& [key, value] : query_params) { + auth_parameter.parameters[key] = UrlUtils::EncodeString(value); + } + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider->MergeAuthHeader(base_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE( + RestHttpClient::Response response, + client->Execute("GET", ResourcePaths::Config(), query_params, headers, "")); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + ConfigResponse config; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, ResourcePaths::Config(), &config)); + merged_options = config.Merge(options); + for (const auto& [key, value] : + RestUtil::ExtractPrefixMap(merged_options, kHeaderOptionPrefix)) { + base_headers[key] = value; + } + std::shared_ptr logger = Logger::GetLogger("RestApi"); + PAIMON_LOG_DEBUG(logger, + "merged %zu client options with the /v1/config response into %zu options", + options.size(), merged_options.size()); + } + std::string prefix; + auto prefix_iter = merged_options.find(kOptionUrlPrefix); + if (prefix_iter != merged_options.end()) { + prefix = prefix_iter->second; + } + return std::unique_ptr(new RestApi(std::move(client), std::move(auth_provider), + base_headers, merged_options, + ResourcePaths(prefix))); +} + +Status RestApi::ErrorToStatus(const RestHttpClient::Response& response) { + // The code of the parsed error body takes precedence over the http status, which + // a gateway may have rewritten. + int64_t code = response.code; + std::string message; + std::string resource_info; + if (!response.body.empty()) { + ErrorResponse error; + if (RapidJsonUtil::FromJsonString(response.body, &error).ok()) { + // The message may embed secrets (e.g. "password=..."), so it is redacted. + message = RestUtil::RedactText(error.GetMessage()); + if (!error.GetResourceType().empty()) { + resource_info = fmt::format(" (resource type: {}, resource name: {})", + error.GetResourceType(), error.GetResourceName()); + } + if (error.GetCode() != 0) { + code = error.GetCode(); + } + } + } + if (message.empty()) { + message = fmt::format("rest server returned http status {}", response.code); + } + message += resource_info; + // Carry the request id through for server-side tracing. + auto request_id_iter = response.headers.find("x-request-id"); + if (request_id_iter == response.headers.end()) { + // Fall back to any header carrying a request id (e.g. x-amz-request-id). + request_id_iter = + std::find_if(response.headers.begin(), response.headers.end(), [](const auto& header) { + return header.first.find("request-id") != std::string::npos; + }); + } + if (request_id_iter != response.headers.end() && !request_id_iter->second.empty() && + request_id_iter->second != "unknown") { + message += fmt::format(" requestId:{}", request_id_iter->second); + } + Status status; + switch (code) { + case HttpStatus::kBadRequest: + status = Status::Invalid(message); + break; + case HttpStatus::kUnauthorized: + status = Status::IOError("not authorized: ", message); + break; + case HttpStatus::kForbidden: + status = Status::IOError("forbidden: ", message); + break; + case HttpStatus::kNotFound: + status = Status::NotExist(message); + break; + case HttpStatus::kConflict: + status = Status::Exist(message); + break; + case HttpStatus::kInternalServerError: + status = Status::IOError("server error: ", message); + break; + case HttpStatus::kNotImplemented: + status = Status::NotImplemented(message); + break; + case HttpStatus::kServiceUnavailable: + status = Status::IOError("service unavailable: ", message); + break; + default: + status = + Status::IOError(fmt::format("rest request failed with code {}: {}", code, message)); + break; + } + return status.WithDetail(std::make_shared(code)); +} + +Result RestApi::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, const std::string& body) const { + RestAuthParameter auth_parameter; + auth_parameter.method = method; + auth_parameter.resource_path = path; + auth_parameter.data = body; + for (const auto& [key, value] : query_params) { + auth_parameter.parameters[key] = UrlUtils::EncodeString(value); + } + StringMap request_headers = base_headers_; + if (!body.empty()) { + // Set before merging the authentication headers, so a provider that signs the + // request headers covers the content type. + request_headers["Content-Type"] = "application/json"; + } + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider_->MergeAuthHeader(request_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + client_->Execute(method, path, query_params, headers, body)); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + return response; +} + +template +Result RestApi::GetEntity(const std::string& path, + const std::map& query_params) const { + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + Execute("GET", path, query_params, "")); + ResponseT entity; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, path, &entity)); + return entity; +} + +template +Result> RestApi::ListAllPages( + const std::string& path) const { + std::vector items; + std::map query_params; + while (true) { + PAIMON_ASSIGN_OR_RAISE(ResponseT response, GetEntity(path, query_params)); + const auto& data = response.Data(); + items.insert(items.end(), data.begin(), data.end()); + const std::optional& next_page_token = response.NextPageToken(); + if (!next_page_token || next_page_token.value().empty() || data.empty()) { + return items; + } + query_params[kQueryParamPageToken] = next_page_token.value(); + } +} + +Result> RestApi::ListDatabases() const { + return ListAllPages(resource_paths_.Databases()); +} + +Status RestApi::CreateDatabase(const std::string& name, + const std::map& options) const { + CreateDatabaseRequest request(name, options); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Databases(), {}, body).status(); +} + +Result RestApi::GetDatabase(const std::string& name) const { + return GetEntity(resource_paths_.Database(name), {}); +} + +Status RestApi::DropDatabase(const std::string& name) const { + return Execute("DELETE", resource_paths_.Database(name), {}, "").status(); +} + +Result> RestApi::ListTables(const std::string& database_name) const { + return ListAllPages(resource_paths_.Tables(database_name)); +} + +Result RestApi::GetTable(const Identifier& identifier) const { + return GetEntity( + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), {}); +} + +Status RestApi::CreateTable(const Identifier& identifier, const std::string& schema_json) const { + CreateTableRequest request(identifier.GetDatabaseName(), identifier.GetTableName(), + schema_json); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Tables(identifier.GetDatabaseName()), {}, body).status(); +} + +Status RestApi::DropTable(const Identifier& identifier) const { + return Execute("DELETE", + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), + {}, "") + .status(); +} + +Status RestApi::RenameTable(const Identifier& from_table, const Identifier& to_table) const { + RenameTableRequest request(from_table.GetDatabaseName(), from_table.GetTableName(), + to_table.GetDatabaseName(), to_table.GetTableName()); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.RenameTable(), {}, body).status(); +} + +Result> RestApi::ListSnapshots(const Identifier& identifier) const { + return ListAllPages( + resource_paths_.Snapshots(identifier.GetDatabaseName(), identifier.GetTableName())); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h new file mode 100644 index 000000000..c4c5f805e --- /dev/null +++ b/src/paimon/rest/rest_api.h @@ -0,0 +1,139 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/snapshot.h" +#include "paimon/rest/resource_paths.h" +#include "paimon/rest/rest_auth.h" +#include "paimon/rest/rest_http_client.h" +#include "paimon/rest/rest_messages.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Carries the code of a failed rest request (the code of the parsed error body, +/// falling back to the http status) so callers can distinguish e.g. authentication +/// failures (401/403) from other IO errors programmatically. +class RestErrorDetail : public StatusDetail { + public: + static constexpr const char* kTypeId = "rest-error-detail"; + + explicit RestErrorDetail(int64_t code) : code_(code) {} + + const char* type_id() const override { + return kTypeId; + } + + std::string ToString() const override { + return "rest error code " + std::to_string(code_); + } + + int64_t GetCode() const { + return code_; + } + + private: + int64_t code_; +}; + +/// The client of the REST catalog server. This layer only talks HTTP + JSON and never +/// touches the file system. +class RestApi { + public: + static constexpr const char* kQueryParamPageToken = "pageToken"; + static constexpr const char* kQueryParamWarehouse = "warehouse"; + /// "prefix" - option key of the url path prefix inserted after "/v1", usually + /// pushed down by "/v1/config". + static constexpr const char* kOptionUrlPrefix = "prefix"; + /// Options with this prefix are sent as http headers (with the prefix stripped). + static constexpr const char* kHeaderOptionPrefix = "header."; + + /// Creates the api client. + /// + /// @param options Client side options; `CatalogOptions::URI` and + /// `CatalogOptions::TOKEN_PROVIDER` are required. + /// @param warehouse Warehouse sent as query parameter of "/v1/config"; may be empty. + /// @param config_required When true, fetch "/v1/config" and merge the server + /// defaults/overrides into `options` with the precedence + /// overrides > client options > defaults. + /// @param http_config Transport level settings, mainly overridable for tests. + static Result> Create( + const std::map& options, const std::string& warehouse, + bool config_required, const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + /// Options merged with the server side config. + const std::map& GetMergedOptions() const { + return options_; + } + + Result> ListDatabases() const; + Status CreateDatabase(const std::string& name, + const std::map& options) const; + Result GetDatabase(const std::string& name) const; + Status DropDatabase(const std::string& name) const; + + Result> ListTables(const std::string& database_name) const; + Result GetTable(const Identifier& identifier) const; + /// `schema_json` uses the schema JSON layout of the protocol + /// (fields/partitionKeys/primaryKeys/options/comment). + Status CreateTable(const Identifier& identifier, const std::string& schema_json) const; + Status DropTable(const Identifier& identifier) const; + Status RenameTable(const Identifier& from_table, const Identifier& to_table) const; + + Result> ListSnapshots(const Identifier& identifier) const; + + /// Maps a non-successful http response to a status: 404 becomes `NotExist`, 409 + /// becomes `Exist`, 400 becomes `Invalid`, 501 becomes `NotImplemented` and the + /// other codes become `IOError`. The status carries a `RestErrorDetail` with the + /// mapped code. + static Status ErrorToStatus(const RestHttpClient::Response& response); + + private: + RestApi(std::unique_ptr client, std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths); + + /// Executes one request with authentication headers and returns the response when it + /// is successful, otherwise the mapped error status. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::string& body) const; + + template + Result GetEntity(const std::string& path, + const std::map& query_params) const; + + /// Fetches all pages of a paged listing api. + template + Result> ListAllPages(const std::string& path) const; + + std::unique_ptr client_; + std::unique_ptr auth_provider_; + /// Base http headers extracted from the "header." options. + std::map base_headers_; + std::map options_; + ResourcePaths resource_paths_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp new file mode 100644 index 000000000..ea53f4a30 --- /dev/null +++ b/src/paimon/rest/rest_auth.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_auth.h" + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" + +namespace paimon { + +Result> BearTokenAuthProvider::MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const { + std::map headers = base_header; + headers["Authorization"] = "Bearer " + token_; + return headers; +} + +Result> AuthProvider::Create( + const std::map& options) { + auto provider_iter = options.find(CatalogOptions::TOKEN_PROVIDER); + if (provider_iter == options.end() || provider_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::TOKEN_PROVIDER)); + } + // Matched leniently in lower case; other clients may match the provider name + // case-sensitively, so only the exact "bear" spelling is portable. + std::string provider = StringUtils::ToLowerCase(provider_iter->second); + if (provider == "bear") { + auto token_iter = options.find(CatalogOptions::TOKEN); + if (token_iter == options.end() || token_iter->second.empty()) { + return Status::Invalid( + fmt::format("option '{}' must be configured for the bear token provider", + CatalogOptions::TOKEN)); + } + return std::make_unique(token_iter->second); + } + return Status::NotImplemented( + fmt::format("unsupported token provider: {}, only 'bear' is supported for now", provider)); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.h b/src/paimon/rest/rest_auth.h new file mode 100644 index 000000000..1cdb67402 --- /dev/null +++ b/src/paimon/rest/rest_auth.h @@ -0,0 +1,66 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Input of one request signature. +struct RestAuthParameter { + std::string method; + std::string resource_path; + /// Query parameters with url-encoded values. + std::map parameters; + /// Request body, empty when the request carries none. + std::string data; +}; + +/// Generates authentication headers for REST catalog requests. +class AuthProvider { + public: + virtual ~AuthProvider() = default; + + /// Returns `base_header` merged with the authentication headers of this provider. + virtual Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const = 0; + + /// Creates the provider configured by `CatalogOptions::TOKEN_PROVIDER`. + static Result> Create( + const std::map& options); +}; + +/// Adds `Authorization: Bearer `. +class BearTokenAuthProvider : public AuthProvider { + public: + explicit BearTokenAuthProvider(const std::string& token) : token_(token) {} + + Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const override; + + private: + std::string token_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp new file mode 100644 index 000000000..dae8d4634 --- /dev/null +++ b/src/paimon/rest/rest_catalog.cpp @@ -0,0 +1,431 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/table/system/system_table_schema.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system.h" +#include "paimon/rest/rest_util.h" +#include "rapidjson/document.h" + +namespace arrow { +class Schema; +} // namespace arrow + +namespace paimon { + +namespace { + +constexpr const char kPathOption[] = "path"; + +/// Maps the default branch "main" (case-insensitive) to "no branch": the default +/// branch is addressed as the bare table. +std::optional NormalizeBranch(std::optional branch) { + if (branch && StringUtils::ToLowerCase(branch.value()) == Identifier::kDefaultMainBranch) { + return std::nullopt; + } + return branch; +} + +/// Builds the identifier sent to the rest server: the system table suffix is stripped +/// while the branch stays in the object name, so the server resolves the branch itself +/// and returns the branch's own schema. The returned table path is the data table +/// root in either case; the branch subdirectory is derived downstream from the +/// branch option (see `ToTableSchema`). +Result ToLoadIdentifier(const Identifier& identifier) { + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + std::string object_name = std::move(data_table_name); + if (branch) { + object_name.append(Identifier::kSystemTableSplitter); + object_name.append(Identifier::kSystemBranchPrefix); + object_name.append(branch.value()); + } + return Identifier(identifier.GetDatabaseName(), object_name); +} + +} // namespace + +RestCatalog::RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse) + : api_(std::move(api)), + fs_(fs), + warehouse_(warehouse), + table_default_options_(RestUtil::ExtractPrefixMap( + api_->GetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), + logger_(Logger::GetLogger("RestCatalog")) {} + +Result> RestCatalog::Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, const RestHttpClient::Config& http_config) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr api, + RestApi::Create(options, warehouse, /*config_required=*/true, http_config)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(api->GetMergedOptions(), file_system)); + return std::unique_ptr( + new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse)); +} + +const std::map& RestCatalog::GetOptions() const { + return api_->GetMergedOptions(); +} + +Status RestCatalog::CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) { + if (CatalogUtils::IsSystemDatabase(name)) { + return Status::Invalid(fmt::format("Cannot create database for system database {}.", name)); + } + Status status = api_->CreateDatabase(name, options); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ListDatabases() const { + return api_->ListDatabases(); +} + +Result RestCatalog::DatabaseExists(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return true; + } + Result response = api_->GetDatabase(db_name); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Status RestCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { + if (CatalogUtils::IsSystemDatabase(name)) { + return Status::Invalid(fmt::format("Cannot drop system database {}.", name)); + } + if (!cascade) { + Result> tables = ListTables(name); + if (!tables.ok()) { + if (tables.status().IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return tables.status(); + } + if (!tables.value().empty()) { + return Status::Invalid( + fmt::format("Cannot drop non-empty database {}. Use cascade=true to force.", name)); + } + } + Status status = api_->DropDatabase(name); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +std::string RestCatalog::GetDatabaseLocation(const std::string& db_name) const { + Result response = api_->GetDatabase(db_name); + if (!response.ok()) { + PAIMON_LOG_WARN(logger_, "failed to get location of database %s: %s", db_name.c_str(), + response.status().ToString().c_str()); + return ""; + } + return response.value().GetLocation(); +} + +Result> RestCatalog::ListTables(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return GlobalSystemTableLoader::GetSupportedTableNames(api_->GetMergedOptions()); + } + return api_->ListTables(db_name); +} + +Status RestCatalog::CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "create table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "create table")); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(c_schema)); + std::map effective_options = options; + for (const auto& [key, value] : table_default_options_) { + effective_options.emplace(key, value); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, schema, partition_keys, + primary_keys, effective_options)); + std::string schema_json; + try { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Document::AllocatorType& allocator = doc.GetAllocator(); + doc.AddMember(rapidjson::StringRef("fields"), + RapidJsonUtil::SerializeValue(table_schema->Fields(), &allocator).Move(), + allocator); + doc.AddMember(rapidjson::StringRef("partitionKeys"), + RapidJsonUtil::SerializeValue(partition_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("primaryKeys"), + RapidJsonUtil::SerializeValue(primary_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(effective_options, &allocator).Move(), + allocator); + schema_json = RestUtil::JsonToString(doc); + } catch (const std::exception& e) { + return Status::SerializationError("failed to serialize create table schema: ", e.what()); + } + Status status = api_->CreateTable(identifier, schema_json); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result RestCatalog::TableExists(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions()); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + Result response = api_->GetTable(load_identifier); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Result RestCatalog::GetTableLocation(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(load_identifier)); + return response.GetPath(); +} + +Status RestCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "drop table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "drop table")); + Status status = api_->DropTable(identifier); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Status RestCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(from_table, "rename table")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(to_table, "rename table")); + Status status = api_->RenameTable(from_table, to_table); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ToTableSchema( + const GetTableResponse& response, const std::optional& branch) { + std::string table_schema_json; + try { + rapidjson::Document src; + src.Parse(response.GetSchemaJson().c_str()); + if (src.HasParseError() || !src.IsObject() || !src.HasMember("fields") || + !src["fields"].IsArray()) { + return Status::Invalid("invalid table schema json from the rest server"); + } + rapidjson::Document out; + out.SetObject(); + rapidjson::Document::AllocatorType& allocator = out.GetAllocator(); + out.AddMember(rapidjson::StringRef("version"), TableSchema::CURRENT_VERSION, allocator); + out.AddMember(rapidjson::StringRef("id"), response.GetSchemaId(), allocator); + PAIMON_ASSIGN_OR_RAISE(int32_t highest_field_id, + TableSchema::ComputeHighestFieldId(src["fields"])); + rapidjson::Value fields(rapidjson::kArrayType); + fields.CopyFrom(src["fields"], allocator); + out.AddMember(rapidjson::StringRef("highestFieldId"), highest_field_id, allocator); + out.AddMember(rapidjson::StringRef("fields"), fields.Move(), allocator); + for (const char* key : {"partitionKeys", "primaryKeys"}) { + rapidjson::Value keys(rapidjson::kArrayType); + if (src.HasMember(key) && src[key].IsArray()) { + keys.CopyFrom(src[key], allocator); + } + out.AddMember(rapidjson::StringRef(key), keys.Move(), allocator); + } + std::map options_map; + if (src.HasMember("options") && src["options"].IsObject()) { + options_map = + RapidJsonUtil::DeserializeValue>(src["options"]); + } + options_map[kPathOption] = response.GetPath(); + response.GetAuditFields().PutAuditOptionsTo(&options_map); + if (branch) { + options_map[Options::BRANCH] = branch.value(); + } + out.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(options_map, &allocator).Move(), allocator); + if (src.HasMember("comment") && src["comment"].IsString()) { + rapidjson::Value comment; + comment.CopyFrom(src["comment"], allocator); + out.AddMember(rapidjson::StringRef("comment"), comment.Move(), allocator); + } + // The server's audit time is used instead of the current time so the + // conversion stays deterministic; 0 (the epoch) when the server did not report + // an "updatedAt". + out.AddMember(rapidjson::StringRef("timeMillis"), + response.GetAuditFields().updated_at.value_or(0), allocator); + table_schema_json = RestUtil::JsonToString(out); + } catch (const std::exception& e) { + return Status::Invalid("failed to convert rest table schema: ", e.what()); + } + return TableSchema::CreateFromJson(table_schema_json); +} + +Result> RestCatalog::LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const { + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(data_identifier)); + if (table_path != nullptr) { + *table_path = response.GetPath(); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr schema, ToTableSchema(response, branch)); + return std::shared_ptr(std::move(schema)); +} + +Result> RestCatalog::LoadTableSchema(const Identifier& identifier) const { + // Serve the global system tables of the "sys" database locally, like + // FileSystemCatalog. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &table_path)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, nullptr)); + return std::static_pointer_cast(schema); +} + +Result> RestCatalog::GetTable(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); + return std::make_shared(schema, identifier.GetDatabaseName(), identifier.GetTableName()); +} + +std::string RestCatalog::GetRootPath() const { + return warehouse_; +} + +std::shared_ptr RestCatalog::GetFileSystem() const { + return fs_; +} + +Result> RestCatalog::ListSnapshots(const Identifier& identifier, + const std::string& branch) const { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "list snapshots")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "list snapshots")); + // The default branch "main" is addressed as the bare table; any other branch is + // sent under its branch object name so the server resolves the branch itself. + std::optional normalized_branch = + NormalizeBranch(branch.empty() ? std::nullopt : std::make_optional(branch)); + std::string object_name = identifier.GetTableName(); + if (normalized_branch) { + object_name.append(Identifier::kSystemTableSplitter); + object_name.append(Identifier::kSystemBranchPrefix); + object_name.append(normalized_branch.value()); + } + Identifier load_identifier(identifier.GetDatabaseName(), object_name); + // The Catalog interface has no pagination, so all pages are fetched. The server + // does not guarantee an ordering across pages while the ListSnapshots contract + // requires ascending ids, hence the sort. + PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, api_->ListSnapshots(load_identifier)); + std::sort(snapshots.begin(), snapshots.end(), + [](const Snapshot& a, const Snapshot& b) { return a.Id() < b.Id(); }); + std::vector result; + result.reserve(snapshots.size()); + for (const auto& snapshot : snapshots) { + result.push_back(snapshot.ToSnapshotInfo()); + } + return result; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h new file mode 100644 index 000000000..94c67ee53 --- /dev/null +++ b/src/paimon/rest/rest_catalog.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_api.h" +#include "paimon/result.h" +#include "paimon/status.h" + +struct ArrowSchema; + +namespace paimon { +class FileSystem; +class TableSchema; + +/// A catalog backed by a REST catalog server. Metadata operations are delegated to +/// `RestApi`; table data is accessed through the file system configured by the +/// (server merged) options. +class RestCatalog : public Catalog { + public: + /// Creates the catalog: fetches and merges "/v1/config" from the server configured + /// by `CatalogOptions::URI`, then builds the file system from the merged options. + /// + /// @param warehouse The warehouse identifier sent to the server; may be empty. + static Result> Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, + const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + Status CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) override; + Status CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) override; + Status DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) override; + Status DropTable(const Identifier& identifier, bool ignore_if_not_exists) override; + Status RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) override; + Result> ListDatabases() const override; + Result> ListTables(const std::string& db_name) const override; + Result DatabaseExists(const std::string& db_name) const override; + Result TableExists(const Identifier& identifier) const override; + std::string GetDatabaseLocation(const std::string& db_name) const override; + Result GetTableLocation(const Identifier& identifier) const override; + Result> LoadTableSchema(const Identifier& identifier) const override; + std::string GetRootPath() const override; + std::shared_ptr GetFileSystem() const override; + Result> GetTable(const Identifier& identifier) const override; + Result> ListSnapshots(const Identifier& identifier, + const std::string& branch) const override; + + /// Options merged with the server side config. + const std::map& GetOptions() const override; + + private: + RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse); + + /// Loads the table from the server and converts the response to a `TableSchema` + /// (options are enriched with the table path, audit info and branch). + Result> LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const; + + static Result> ToTableSchema( + const GetTableResponse& response, const std::optional& branch); + + std::unique_ptr api_; + std::shared_ptr fs_; + std::string warehouse_; + /// The "table-default." options of the merged config, applied to `CreateTable` + /// options when absent. + std::map table_default_options_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp new file mode 100644 index 000000000..239750e5a --- /dev/null +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -0,0 +1,997 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/rest/rest_api.h" +#include "paimon/schema/schema.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +constexpr const char kToken[] = "test-token"; +constexpr const char kPrefix[] = "paimon"; +constexpr const char kWarehouse[] = "wh1"; + +/// The in-memory catalog state behind the mock rest server. +struct MockCatalogState { + struct TableData { + std::string schema_json; + int64_t schema_id = 0; + std::string path; + }; + // database name -> table name -> table + std::map> databases; + // headers of the last request, with lower-cased names + std::map last_headers; + // when set, every request except "/v1/config" fails with this http code + std::optional force_error_code; + // guards all fields above: the handler runs on the mock server's accept thread + // while tests seed and inspect the state from the test thread + std::mutex mutex; +}; + +MockRestServer::Response JsonResponse(int32_t code, const std::string& body) { + MockRestServer::Response response; + response.code = code; + response.body = body; + return response; +} + +MockRestServer::Response MockError(int32_t code, const std::string& resource_type, + const std::string& resource_name, const std::string& message) { + ErrorResponse error(resource_type, resource_name, message, code); + return JsonResponse(code, error.ToJsonString().value()); +} + +std::string SnapshotJson(int64_t id) { + return fmt::format( + R"({{"version":3,"id":{},"schemaId":0,"baseManifestList":"bml","deltaManifestList":"dml",)" + R"("commitUser":"user1","commitIdentifier":1,"commitKind":"APPEND","timeMillis":100,)" + R"("totalRecordCount":10,"deltaRecordCount":1}})", + id); +} + +std::string TableResponseJson(const std::string& name, const MockCatalogState::TableData& table) { + return fmt::format( + R"({{"id":"1","name":"{}","path":"{}","isExternal":false,"schemaId":{},"schema":{},)" + R"("owner":"owner1","updatedAt":123}})", + name, table.path, table.schema_id, table.schema_json); +} + +/// Serves `names` one item per page to exercise the pagination loop of the client. +std::pair, std::optional> PageOf( + const std::vector& names, const MockRestServer::Request& request) { + size_t index = 0; + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter != request.query_params.end()) { + // The exception-free parse keeps a malformed token from terminating the test + // binary: the handler runs on the mock server's accept thread. + index = static_cast( + StringUtils::StringToValue(token_iter->second).value_or(0)); + } + std::vector page; + std::optional next_page_token; + if (index < names.size()) { + page.push_back(names[index]); + if (index + 1 < names.size()) { + next_page_token = std::to_string(index + 1); + } + } + return {page, next_page_token}; +} + +/// Implements the subset of the rest catalog protocol used by `RestCatalog` on top of +/// `MockCatalogState`. +MockRestServer::Response HandleCatalogRequest(MockCatalogState* state, + const MockRestServer::Request& request) { + std::lock_guard lock(state->mutex); + state->last_headers = request.headers; + auto auth_iter = request.headers.find("authorization"); + if (auth_iter == request.headers.end() || + auth_iter->second != std::string("Bearer ") + kToken) { + return MockError(401, "", "", "invalid token"); + } + if (request.path == "/v1/config") { + auto warehouse_iter = request.query_params.find("warehouse"); + if (warehouse_iter == request.query_params.end() || warehouse_iter->second != kWarehouse) { + return MockError(400, "", "", "unexpected warehouse"); + } + ConfigResponse config( + {{RestApi::kOptionUrlPrefix, kPrefix}, + {"header.x-server-header", "from-config"}, + {"table-default.write-only", "true"}, + {"table-default.bucket", "8"}}, + {{"server-override", "from-server"}, {"header.x-shared-header", "from-config"}}); + return JsonResponse(200, config.ToJsonString().value()); + } + if (state->force_error_code) { + return MockError(state->force_error_code.value(), "", "", "injected failure"); + } + const std::string base = std::string("/v1/") + kPrefix; + if (request.path.rfind(base, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string rest = request.path.substr(base.size()); + + if (rest == "/databases") { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, tables] : state->databases) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListDatabasesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateDatabaseRequest create_request("", {}); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create database request"); + } + if (state->databases.count(create_request.GetName()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeDatabase, + create_request.GetName(), "database already exists"); + } + state->databases[create_request.GetName()] = {}; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + if (rest == "/tables/rename" && request.method == "POST") { + RenameTableRequest rename_request("", "", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &rename_request).ok()) { + return MockError(400, "", "", "bad rename table request"); + } + auto db_iter = state->databases.find(rename_request.GetSourceDatabase()); + if (db_iter == state->databases.end() || + db_iter->second.count(rename_request.GetSourceTable()) == 0) { + return MockError(404, ErrorResponse::kResourceTypeTable, + rename_request.GetSourceTable(), "table not found"); + } + auto& dest_tables = state->databases[rename_request.GetDestinationDatabase()]; + if (dest_tables.count(rename_request.GetDestinationTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, + rename_request.GetDestinationTable(), "table already exists"); + } + dest_tables[rename_request.GetDestinationTable()] = + db_iter->second[rename_request.GetSourceTable()]; + db_iter->second.erase(rename_request.GetSourceTable()); + return JsonResponse(200, ""); + } + + const std::string databases_prefix = "/databases/"; + if (rest.rfind(databases_prefix, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string remainder = rest.substr(databases_prefix.size()); + size_t tables_pos = remainder.find("/tables"); + + if (tables_pos == std::string::npos) { + const std::string& db_name = remainder; + auto db_iter = state->databases.find(db_name); + if (request.method == "GET") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + std::string body = fmt::format( + R"({{"id":"1","name":"{}","location":"{}/{}.db","options":{{"dbk":"dbv"}}}})", + db_name, kWarehouse, db_name); + return JsonResponse(200, body); + } + if (request.method == "DELETE") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + state->databases.erase(db_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + std::string db_name = remainder.substr(0, tables_pos); + auto db_iter = state->databases.find(db_name); + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, "database not found"); + } + auto& tables = db_iter->second; + std::string table_part = remainder.substr(tables_pos + std::strlen("/tables")); + + if (table_part.empty()) { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, table] : tables) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListTablesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateTableRequest create_request("", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create table request"); + } + if (tables.count(create_request.GetTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, create_request.GetTable(), + "table already exists"); + } + MockCatalogState::TableData table; + table.schema_json = create_request.GetSchemaJson(); + table.schema_id = 0; + table.path = fmt::format("{}/{}.db/{}", kWarehouse, db_name, create_request.GetTable()); + tables[create_request.GetTable()] = table; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + // "/{table}" or "/{table}/snapshots" + std::string table_name = table_part.substr(1); + bool list_snapshots = false; + const std::string snapshots_suffix = "/snapshots"; + if (table_name.size() > snapshots_suffix.size() && + table_name.compare(table_name.size() - snapshots_suffix.size(), snapshots_suffix.size(), + snapshots_suffix) == 0) { + table_name = table_name.substr(0, table_name.size() - snapshots_suffix.size()); + list_snapshots = true; + } + auto table_iter = tables.find(table_name); + if (table_iter == tables.end()) { + return MockError(404, ErrorResponse::kResourceTypeTable, table_name, "table not found"); + } + if (list_snapshots) { + // two pages, out of order to exercise pagination and sorting + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter == request.query_params.end()) { + return JsonResponse( + 200, fmt::format(R"({{"snapshots":[{}],"nextPageToken":"1"}})", SnapshotJson(2))); + } + return JsonResponse(200, fmt::format(R"({{"snapshots":[{}]}})", SnapshotJson(1))); + } + if (request.method == "GET") { + return JsonResponse(200, TableResponseJson(table_name, table_iter->second)); + } + if (request.method == "DELETE") { + tables.erase(table_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); +} + +} // namespace + +class RestCatalogTest : public ::testing::Test { + protected: + void SetUp() override { + state_ = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + server_, MockRestServer::Start([state = state_](const MockRestServer::Request& req) { + return HandleCatalogRequest(state.get(), req); + })); + options_ = { + {CatalogOptions::METASTORE, "rest"}, + {CatalogOptions::URI, server_->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + {Options::FILE_SYSTEM, "local"}, + // mock_format is linked statically into the test binary, so its factory is + // registered in the binary's own registry even when the real format plugin + // dylibs register into a different one (macOS two-level namespace) + {Options::FILE_FORMAT, "mock_format"}, + {Options::MANIFEST_FORMAT, "mock_format"}, + {"header.x-client-header", "from-client"}, + {"header.x-shared-header", "from-client"}, + }; + } + + void TearDown() override { + if (server_) { + server_->Stop(); + } + } + + Result> CreateRestCatalog() { + return RestCatalog::Create(kWarehouse, options_, nullptr); + } + + Status CreateSampleTable(Catalog* catalog, const Identifier& identifier, + bool ignore_if_exists = false) { + std::shared_ptr schema = + arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::utf8())}); + struct ArrowSchema c_schema; + if (!arrow::ExportSchema(*schema, &c_schema).ok()) { + return Status::Invalid("failed to export arrow schema"); + } + Status status = + catalog->CreateTable(identifier, &c_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, {{"bucket", "2"}}, ignore_if_exists); + // CreateTable takes ownership of the exported schema only once it reaches + // arrow::ImportSchema, which an identifier rejected by its checks never does + if (c_schema.release != nullptr) { + c_schema.release(&c_schema); + } + return status; + } + + /// Seeds `schema_json` as table `table_name` of "db1" behind the mock server and + /// expects loading the table to fail with an Invalid status carrying + /// `expected_message`. + void ExpectBrokenSchemaRejected(Catalog* catalog, const std::string& table_name, + const std::string& schema_json, + const std::string& expected_message) { + MockCatalogState::TableData table_data; + table_data.schema_json = schema_json; + table_data.path = "wh1/db1.db/" + table_name; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"][table_name] = table_data; + } + Status status = catalog->GetTable(Identifier("db1", table_name)).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, expected_message); + } + + std::shared_ptr state_; + std::unique_ptr server_; + std::map options_; +}; + +TEST_F(RestCatalogTest, CreateMergesServerConfig) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + const std::map& merged = catalog->GetOptions(); + ASSERT_EQ(kPrefix, merged.at(RestApi::kOptionUrlPrefix)); + ASSERT_EQ("from-server", merged.at("server-override")); + ASSERT_EQ(kWarehouse, catalog->GetRootPath()); + ASSERT_NE(nullptr, catalog->GetFileSystem()); +} + +TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { + // "metastore=rest" selects the rest catalog + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_TRUE(databases.empty()); + // an unknown metastore is rejected + options_[CatalogOptions::METASTORE] = "something-else"; + Status status = Catalog::Create(kWarehouse, options_).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, "unsupported metastore"); +} + +TEST_F(RestCatalogTest, CreateWithWrongTokenFails) { + options_[CatalogOptions::TOKEN] = "wrong-token"; + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "not authorized"); +} + +TEST_F(RestCatalogTest, CreateRejectsInvalidOptions) { + // all rejected by client side validation, before any request reaches the server + const std::map valid_options = options_; + + options_.erase(CatalogOptions::URI); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'uri' must be configured"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN_PROVIDER); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'token.provider' must be configured"); + + options_ = valid_options; + options_[CatalogOptions::TOKEN_PROVIDER] = "dlf"; + Status unsupported_provider = CreateRestCatalog().status(); + ASSERT_TRUE(unsupported_provider.IsNotImplemented()) << unsupported_provider.ToString(); + ASSERT_NOK_WITH_MSG(unsupported_provider, "unsupported token provider"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "bear token provider"); +} + +TEST_F(RestCatalogTest, DatabaseOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateDatabase("db2", {}, /*ignore_if_exists=*/false)); + Status duplicated = catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/true)); + + // the mock server returns one database per page + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_EQ((std::vector{"db1", "db2"}), databases); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("db1")); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db3")); + ASSERT_FALSE(exists); + + ASSERT_EQ("wh1/db1.db", catalog->GetDatabaseLocation("db1")); + ASSERT_EQ("", catalog->GetDatabaseLocation("db3")); + + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, /*cascade=*/false)); + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/true, /*cascade=*/false)); + Status missing = catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, + /*cascade=*/false); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_NOK_WITH_MSG( + catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/false), + "non-empty database"); + // cascade drop skips the emptiness check + ASSERT_OK(catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/true)); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db1")); + ASSERT_FALSE(exists); +} + +TEST_F(RestCatalogTest, TableOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + + Status missing_db = CreateSampleTable(catalog.get(), Identifier("db_missing", "t1")); + ASSERT_TRUE(missing_db.IsNotExist()) << missing_db.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + Status duplicated = CreateSampleTable(catalog.get(), identifier); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier, /*ignore_if_exists=*/true)); + + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1"}), tables); + Status list_missing = catalog->ListTables("db_missing").status(); + ASSERT_TRUE(list_missing.IsNotExist()) << list_missing.ToString(); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); + ASSERT_EQ("t1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); + ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); + ASSERT_EQ(0, schema->Id()); + // options are enriched with the table path and audit info from the server + ASSERT_EQ("wh1/db1.db/t1", schema->Options().at("path")); + ASSERT_EQ("owner1", schema->Options().at("owner")); + + // "table-default." options of the merged config apply only where the caller left the + // option unset: "write-only" is taken from the config, "bucket" keeps the value passed + // to CreateTable instead of the configured "table-default.bucket" of 8 + ASSERT_EQ("true", schema->Options().at("write-only")); + ASSERT_EQ("2", schema->Options().at("bucket")); + + // timeMillis is backed by the server's audit "updatedAt" instead of the current + // time, keeping the conversion deterministic + std::shared_ptr table_schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, table_schema); + ASSERT_EQ(123, table_schema->TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_schema, + catalog->LoadTableSchema(identifier)); + ASSERT_EQ((std::vector{"f0", "f1"}), loaded_schema->FieldNames()); + Status schema_missing = catalog->LoadTableSchema(Identifier("db1", "t2")).status(); + ASSERT_TRUE(schema_missing.IsNotExist()) << schema_missing.ToString(); + + // a schema comment of the server response is carried into the table schema + MockCatalogState::TableData commented; + commented.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {},)" + R"( "comment": "a table comment"})"; + commented.path = "wh1/db1.db/commented"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["commented"] = commented; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, + catalog->GetTable(Identifier("db1", "commented"))); + ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false)); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_TRUE(exists); + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/true)); + Status rename_missing = catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(rename_missing.IsNotExist()) << rename_missing.ToString(); + + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/true)); + Status drop_missing = catalog->DropTable(Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(drop_missing.IsNotExist()) << drop_missing.ToString(); +} + +TEST_F(RestCatalogTest, ClientAndServerHeadersAreSent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + // "header." options from both the client and the merged server config are sent as + // http headers on every request + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("from-client", state_->last_headers.at("x-client-header")); + ASSERT_EQ("from-config", state_->last_headers.at("x-server-header")); + // when the client and the server config set the same "header." option, the + // merged config wins (overrides > client options > defaults) + ASSERT_EQ("from-config", state_->last_headers.at("x-shared-header")); + ASSERT_EQ(std::string("Bearer ") + kToken, state_->last_headers.at("authorization")); + } + // a request carrying a body declares the json content type (set before the auth + // headers are merged, so a signing auth provider covers it) + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("application/json", state_->last_headers.at("content-type")); + } +} + +TEST_F(RestCatalogTest, SystemTableSchema) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the "options" system table has a static schema and needs no file system access + Identifier system_identifier("db1", "t1$options"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(system_identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t9$options"))); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(system_identifier)); + ASSERT_EQ((std::vector{"key", "value"}), schema->FieldNames()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); + ASSERT_EQ("t1$options", table->Name()); + ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); + + Status unsupported = catalog->LoadTableSchema(Identifier("db1", "t1$unsupported")).status(); + ASSERT_TRUE(unsupported.IsNotExist()) << unsupported.ToString(); +} + +TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the branch is seeded under its object name with a schema diverged from the main + // table: the identifier sent to the server keeps the branch, and the server + // resolves it. The path it reports is the data table root (not the branch + // subdirectory): downstream readers derive "/branch/branch-" from the + // branch option, so a branch path here would be applied twice + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "b0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.schema_id = 3; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + + Identifier branch_identifier("db1", "t1$branch_b1"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(branch_identifier)); + ASSERT_TRUE(exists); + // a branch the server does not know is missing instead of silently falling back to + // the main table + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$branch_missing"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(branch_identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); + ASSERT_EQ("t1$branch_b1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + // the branch's own schema and schema id, not the main table's + ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); + ASSERT_EQ(3, schema->Id()); + ASSERT_EQ("b1", schema->Options().at(Options::BRANCH)); + + // the default branch is addressed as the bare table: "t1$branch_MAIN" resolves to + // "t1" (case-insensitively) and carries no branch option + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, + catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); + std::shared_ptr main_schema = + std::dynamic_pointer_cast(main_table->LatestSchema()); + ASSERT_NE(nullptr, main_schema); + ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); + ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); + + // a system table on a branch resolves against the branch's data table: the system + // suffix is stripped while the branch stays in the identifier sent to the server + ASSERT_OK_AND_ASSIGN(std::shared_ptr options_schema, + catalog->LoadTableSchema(Identifier("db1", "t1$branch_b1$options"))); + ASSERT_EQ((std::vector{"key", "value"}), options_schema->FieldNames()); + // a missing branch fails through the system table path too instead of silently + // resolving against the main table + Status missing_branch = + catalog->LoadTableSchema(Identifier("db1", "t1$branch_missing$options")).status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + // writing operations reject branch tables + ASSERT_NOK_WITH_MSG(catalog->DropTable(branch_identifier, /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(catalog->RenameTable(branch_identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(CreateSampleTable(catalog.get(), Identifier("db1", "t2$branch_b1")), + "branch table"); +} + +TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + // inject a table with nested types directly; ids inside nested rows must be + // included when computing highestFieldId + MockCatalogState::TableData table_data; + table_data.schema_json = R"({ + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 3, "name": "inner", "type": "INT"}]}}, + {"id": 2, "name": "arr", "type": {"type": "ARRAY", + "element": {"type": "ROW", + "fields": [{"id": 7, "name": "deep", "type": "BIGINT"}]}}}, + {"id": 4, "name": "m", "type": {"type": "MAP", + "key": {"type": "ROW NOT NULL", + "fields": [{"id": 8, "name": "k", "type": "INT NOT NULL"}]}, + "value": {"type": "ROW", + "fields": [{"id": 9, "name": "v", "type": "INT"}]}}} + ], + "partitionKeys": [], + "primaryKeys": [], + "options": {} + })"; + table_data.schema_id = 5; + table_data.path = "wh1/db1.db/nested"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["nested"] = table_data; + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("db1", "nested"))); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ(5, schema->Id()); + // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value + // must all be traversed + ASSERT_EQ(9, schema->HighestFieldId()); + ASSERT_EQ((std::vector{"f0", "s", "arr", "m"}), schema->FieldNames()); +} + +TEST_F(RestCatalogTest, ListTablesPaged) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t2"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t3"))); + // the mock server returns one table per page + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1", "t2", "t3"}), tables); +} + +TEST_F(RestCatalogTest, BrokenSchemaRejected) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + // a broken schema with duplicated field ids from the server must be rejected + ExpectBrokenSchemaRejected( + catalog.get(), "duplicate", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 0, "name": "f1", "type": "STRING"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // an id inside a nested row colliding with an outer field id is a duplicate too + ExpectBrokenSchemaRejected( + catalog.get(), "duplicate_nested", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 1, "name": "s", "type": {"type": "ROW",)" + R"( "fields": [{"id": 0, "name": "inner", "type": "INT"}]}}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // a field without an integer id must fail the conversion instead of being + // silently skipped when computing highestFieldId + ExpectBrokenSchemaRejected(catalog.get(), "no_id", + R"({"fields": [{"name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + ExpectBrokenSchemaRejected( + catalog.get(), "string_id", + R"({"fields": [{"id": "0", "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + // a field that is not an object fails too instead of being silently skipped + ExpectBrokenSchemaRejected(catalog.get(), "non_object", + R"({"fields": [1],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "must be an object"); +} + +TEST_F(RestCatalogTest, ServerErrorIsPropagatedNotMappedToAbsent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + { + std::lock_guard lock(state_->mutex); + state_->force_error_code = 500; + } + // a server failure surfaces as an error instead of "does not exist" + Status db_status = catalog->DatabaseExists("db1").status(); + ASSERT_NOK_WITH_MSG(db_status, "server error"); + Status table_status = catalog->TableExists(Identifier("db1", "t1")).status(); + ASSERT_NOK_WITH_MSG(table_status, "server error"); +} + +TEST_F(RestCatalogTest, SystemTableChecks) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_NOK_WITH_MSG(catalog->CreateDatabase("sys", {}, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropDatabase("sys", false, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropTable(Identifier("sys", "t"), false), "system table"); + ASSERT_NOK_WITH_MSG( + catalog->RenameTable(Identifier("db1", "t1$snapshots"), Identifier("db1", "t2"), false), + "system table"); +} + +TEST_F(RestCatalogTest, SystemDatabase) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + // the "sys" database and its global system tables are resolved locally without + // contacting the server, like in FileSystemCatalog + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("sys")); + ASSERT_TRUE(exists); + + ASSERT_OK_AND_ASSIGN(std::vector sys_tables, catalog->ListTables("sys")); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "tables"))); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(Identifier("sys", "tables"))); + ASSERT_FALSE(schema->FieldNames().empty()); + Status missing = catalog->LoadTableSchema(Identifier("sys", "unsupported")).status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); +} + +TEST_F(RestCatalogTest, ListSnapshots) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + + ASSERT_OK_AND_ASSIGN(std::vector snapshots, + catalog->ListSnapshots(identifier, "")); + ASSERT_EQ(2, snapshots.size()); + // fetched via two pages and sorted by snapshot id + ASSERT_EQ(1, snapshots[0].snapshot_id); + ASSERT_EQ(2, snapshots[1].snapshot_id); + ASSERT_EQ("user1", snapshots[0].commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, snapshots[0].commit_kind); + + // the default branch is addressed as the bare table, so passing it explicitly + // (case-insensitively) equals the branch-less call + ASSERT_OK_AND_ASSIGN(std::vector main_snapshots, + catalog->ListSnapshots(identifier, "MAIN")); + ASSERT_EQ(2, main_snapshots.size()); + + Status missing = catalog->ListSnapshots(Identifier("db1", "t9"), "").status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + // a non-main branch is sent under its branch object name, so the server resolves + // the branch and lists its own snapshots; the reported path is the data table + // root, the branch subdirectory is derived from the branch option downstream + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + ASSERT_OK_AND_ASSIGN(std::vector branch_snapshots, + catalog->ListSnapshots(identifier, "b1")); + ASSERT_EQ(2, branch_snapshots.size()); + // a branch the server does not know is missing instead of silently falling back + // to the main table + Status missing_branch = catalog->ListSnapshots(identifier, "b_missing").status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + // a branch must be passed as the branch argument, not encoded in the identifier + ASSERT_NOK_WITH_MSG(catalog->ListSnapshots(Identifier("db1", "t1$branch_b1"), "").status(), + "branch table"); +} + +TEST(RestApiErrorTest, ErrorToStatus) { + RestHttpClient::Response response; + response.code = 404; + response.body = R"({"message": "no table", "resourceType": "TABLE", "resourceName": "t1"})"; + response.headers["x-request-id"] = "req-123"; + Status status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(status.IsNotExist()); + ASSERT_TRUE(status.ToString().find("requestId:req-123") != std::string::npos) + << status.ToString(); + + response.code = 409; + Status exist_status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(exist_status.IsExist()); + + response.code = 501; + response.body = ""; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotImplemented()); + + response.code = 500; + response.body = "not-a-json"; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "server error"); + + // the code of the error body wins over the http status when they disagree (e.g. a + // gateway rewriting the status) + response.code = 500; + response.body = R"({"message": "gone", "code": 404})"; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotExist()); + + // an empty server message still yields a status naming the http code, and the + // resource info is kept + response.code = 404; + response.body = R"({"resourceType": "TABLE", "resourceName": "t1"})"; + Status empty_message = RestApi::ErrorToStatus(response); + ASSERT_TRUE(empty_message.ToString().find("http status 404") != std::string::npos) + << empty_message.ToString(); + ASSERT_TRUE(empty_message.ToString().find("resource name: t1") != std::string::npos) + << empty_message.ToString(); + + // server messages that may embed secrets are redacted as a whole + response.code = 400; + response.body = R"({"message": "bad option password=abc123", "code": 400})"; + Status redacted = RestApi::ErrorToStatus(response); + ASSERT_TRUE(redacted.IsInvalid()) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("abc123") == std::string::npos) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("******") != std::string::npos) << redacted.ToString(); + + // any header carrying a request id is used when x-request-id is absent + response.code = 404; + response.body = ""; + response.headers.clear(); + response.headers["x-amz-request-id"] = "amz-1"; + Status fallback = RestApi::ErrorToStatus(response); + ASSERT_TRUE(fallback.ToString().find("requestId:amz-1") != std::string::npos) + << fallback.ToString(); + + // the "unknown" placeholder is not a real request id + response.headers.clear(); + response.headers["x-request-id"] = "unknown"; + Status unknown_id = RestApi::ErrorToStatus(response); + ASSERT_TRUE(unknown_id.ToString().find("requestId") == std::string::npos) + << unknown_id.ToString(); + + // 401/403 map to IOError; the mapped code is carried as a status detail so + // callers can distinguish them + response.headers.clear(); + response.code = 401; + Status not_authorized = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(not_authorized, "not authorized"); + ASSERT_NE(nullptr, not_authorized.detail()); + ASSERT_EQ(std::string(RestErrorDetail::kTypeId), not_authorized.detail()->type_id()); + ASSERT_EQ(401, std::static_pointer_cast(not_authorized.detail())->GetCode()); + response.code = 403; + Status forbidden = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(forbidden, "forbidden"); + ASSERT_EQ(403, std::static_pointer_cast(forbidden.detail())->GetCode()); + + // 503 and the codes without an own mapping (e.g. 429) become IOError with a + // message naming the code + response.code = 503; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "service unavailable"); + response.code = 429; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 429"); + response.code = 418; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 418"); +} + +TEST(RestApiErrorTest, MalformedSuccessBodyFails) { + // a 200 response whose body is not the expected json must fail, not crash or + // return partial data + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = "not-a-json"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + Status list_status = api->ListDatabases().status(); + ASSERT_NOK(list_status); + // the body is not echoed into the error: a successful response may contain + // credentials + ASSERT_EQ(std::string::npos, list_status.ToString().find("not-a-json")) + << list_status.ToString(); + ASSERT_NOK(api->GetTable(Identifier("db1", "t1")).status()); + + // the same garbage on "/v1/config" fails the creation with the config fetch + Status config_status = RestApi::Create(options, "", /*config_required=*/true).status(); + ASSERT_NOK(config_status); + ASSERT_EQ(std::string::npos, config_status.ToString().find("not-a-json")) + << config_status.ToString(); +} + +TEST(RestApiErrorTest, PagedListingStopsOnEmptyPageWithToken) { + // a server that keeps returning a page token with no data must not loop forever + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"databases":[],"nextPageToken":"more"})"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + ASSERT_OK_AND_ASSIGN(std::vector databases, api->ListDatabases()); + ASSERT_TRUE(databases.empty()); + ASSERT_EQ(1, request_count.load()); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp new file mode 100644 index 000000000..185b40228 --- /dev/null +++ b/src/paimon/rest/rest_http_client.cpp @@ -0,0 +1,362 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "curl/curl.h" +#include "fmt/format.h" +#include "paimon/common/utils/http_client.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { + +size_t WriteBodyCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* body = static_cast(user_data); + body->append(data, size * nmemb); + return size * nmemb; +} + +size_t WriteHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* headers = static_cast(user_data); + size_t total = size * nmemb; + // A status line starts the header block of the next response of the connection + // (e.g. after a followed redirect): the headers of the previous response are + // dropped so that only the final response's headers are reported. + if (total >= 5 && std::strncmp(data, "HTTP/", 5) == 0) { + headers->clear(); + } + ParseHttpHeaderLine(data, total, headers); + return total; +} + +// Of the supported methods (see `Execute`), only GET and DELETE are idempotent. +bool IsIdempotent(const std::string& method) { + return method == "GET" || method == "DELETE"; +} + +bool IsRetriableCode(int64_t code) { + return code == HttpStatus::kTooManyRequests || code == HttpStatus::kServiceUnavailable; +} + +// Parses the RFC 1123 form of an http date, e.g. "Wed, 21 Oct 2015 07:28:00 GMT", +// into unix epoch seconds. The names are matched against fixed English tables +// independently of the process locale. +std::optional ParseHttpDateSeconds(const std::string& value) { + // ", :: GMT" + std::vector parts = StringUtils::Split(value, " ", /*ignore_empty=*/true); + if (parts.size() != 6 || parts[5] != "GMT" || parts[0].size() != 4 || parts[0][3] != ',') { + return std::nullopt; + } + static constexpr const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int64_t month = 0; + for (size_t i = 0; i < 12; i++) { + if (parts[2] == kMonths[i]) { + month = static_cast(i) + 1; + break; + } + } + std::optional day = StringUtils::StringToValue(parts[1]); + std::optional year = StringUtils::StringToValue(parts[3]); + std::vector time_parts = StringUtils::Split(parts[4], ":", + /*ignore_empty=*/false); + if (month == 0 || !day || !year || time_parts.size() != 3) { + return std::nullopt; + } + std::optional hour = StringUtils::StringToValue(time_parts[0]); + std::optional minute = StringUtils::StringToValue(time_parts[1]); + std::optional second = StringUtils::StringToValue(time_parts[2]); + if (!hour || !minute || !second || day.value() < 1 || day.value() > 31 || year.value() < 1970 || + year.value() > 9999 || hour.value() > 23 || minute.value() > 59 || second.value() > 60) { + return std::nullopt; + } + // Days between 1970-01-01 and the given civil date ("days from civil" algorithm). + int64_t y = year.value() - (month <= 2 ? 1 : 0); + int64_t era = y / 400; + int64_t yoe = y - era * 400; + int64_t doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day.value() - 1; + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + int64_t days = era * 146097 + doe - 719468; + return days * 86400 + hour.value() * 3600 + minute.value() * 60 + second.value(); +} + +// Returns false for the transport failure kinds that do not become healthy by +// retrying: name resolution and connect failures, timeouts, TLS failures, and the +// server closing the connection without a response (CURLE_GOT_NOTHING). +// +// A response truncated mid-body (CURLE_PARTIAL_FILE) stays retriable: the request +// demonstrably reached a live server, so retrying the idempotent methods is safe +// and likely to succeed. +bool IsRetriableTransportError(CURLcode code) { + switch (code) { + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_CONNECT: + case CURLE_OPERATION_TIMEDOUT: + case CURLE_GOT_NOTHING: + case CURLE_SSL_CONNECT_ERROR: + case CURLE_SSL_CERTPROBLEM: + case CURLE_SSL_CIPHER: + case CURLE_PEER_FAILED_VERIFICATION: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_SSL_ISSUER_ERROR: + return false; + default: + return true; + } +} + +} // namespace + +RestHttpClient::RestHttpClient(const std::string& base_uri, const Config& config) + : curl_guard_(EnsureCurlGlobalInit()), + base_uri_(base_uri), + config_(config), + logger_(Logger::GetLogger("RestHttpClient")) {} + +Result> RestHttpClient::Create(const std::string& base_uri) { + return Create(base_uri, Config()); +} + +Result> RestHttpClient::Create(const std::string& base_uri, + const Config& config) { + if (base_uri.empty()) { + return Status::Invalid("uri of the http client is empty"); + } + return std::unique_ptr(new RestHttpClient(NormalizeUri(base_uri), config)); +} + +std::string RestHttpClient::NormalizeUri(const std::string& uri) { + std::string normalized = uri; + StringUtils::Trim(&normalized); + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (normalized.rfind("http://", 0) != 0 && normalized.rfind("https://", 0) != 0) { + normalized = "http://" + normalized; + } + return normalized; +} + +std::string RestHttpClient::BuildQueryString( + const std::map& query_params) { + std::string query; + for (const auto& [key, value] : query_params) { + if (!query.empty()) { + query.push_back('&'); + } + query.append(UrlUtils::EncodeString(key)); + query.push_back('='); + query.append(UrlUtils::EncodeString(value)); + } + return query; +} + +Result RestHttpClient::ExecuteOnce( + const std::string& method, const std::string& url, + const std::map& headers, const std::string& body, + bool* transport_retriable) const { + CURL* curl = curl_easy_init(); + if (curl == nullptr) { + return Status::IOError("failed to create curl handle"); + } + ScopeGuard cleanup_curl([curl] { curl_easy_cleanup(curl); }); + Response response; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + static_cast(config_.connect_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, + static_cast(config_.request_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteBodyCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response.headers); + // Follow redirects transparently, restricted to http(s) targets. + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L); +#if CURL_AT_LEAST_VERSION(7, 85, 0) + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); +#else + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); +#endif + + if (method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } else if (method == "DELETE") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (!body.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } + } + + struct curl_slist* header_list = nullptr; + ScopeGuard free_headers([&header_list] { curl_slist_free_all(header_list); }); + auto append_header = [&header_list](const std::string& line) -> bool { + struct curl_slist* updated_list = curl_slist_append(header_list, line.c_str()); + if (updated_list == nullptr) { + return false; + } + header_list = updated_list; + return true; + }; + // Disable the automatic "Expect: 100-continue" behavior of libcurl on POST. + bool headers_ok = append_header("Expect:"); + for (const auto& [name, value] : headers) { + headers_ok = headers_ok && append_header(name + ": " + value); + } + if (!headers_ok) { + return Status::IOError("failed to create http headers"); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); + + CURLcode curl_code = curl_easy_perform(curl); + if (curl_code != CURLE_OK) { + *transport_retriable = IsRetriableTransportError(curl_code); + // Do not include the response body or full url in errors, they may carry + // sensitive information such as credentials. + return Status::IOError( + fmt::format("http {} request failed: {}", method, curl_easy_strerror(curl_code))); + } + // curl writes a `long` through the pointer, so the type cannot be narrowed here. + // NOLINTNEXTLINE(google-runtime-int) + long http_code = 0; // NOLINT(runtime/int) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + response.code = http_code; + return response; +} + +int64_t RestHttpClient::ComputeRetryDelayMs(const Config& config, int32_t execution_count, + const Response* response, int64_t now_epoch_seconds) { + if (response != nullptr) { + auto iter = response->headers.find("retry-after"); + if (iter != response->headers.end()) { + std::optional retry_after_ms; + std::optional retry_after_seconds = + StringUtils::StringToValue(iter->second); + if (retry_after_seconds) { + // Clamped so that a misbehaving server cannot overflow the conversion. + constexpr int64_t kMaxSeconds = std::numeric_limits::max() / 1000; + retry_after_ms = + std::clamp(retry_after_seconds.value(), -kMaxSeconds, kMaxSeconds) * 1000; + } else { + std::optional date_seconds = ParseHttpDateSeconds(iter->second); + if (date_seconds) { + retry_after_ms = (date_seconds.value() - now_epoch_seconds) * 1000; + } + } + // Non-positive values fall through to the exponential backoff. + if (retry_after_ms && retry_after_ms.value() > 0) { + return retry_after_ms.value(); + } + } + } + int64_t multiplier = static_cast(1) + << std::min(execution_count - 1, static_cast(6)); + int64_t delay_ms = config.retry_base_delay_ms * multiplier; + if (delay_ms > 0) { + static thread_local std::mt19937 generator( + std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution jitter(0, delay_ms / 10); + delay_ms += jitter(generator); + } + return delay_ms; +} + +int64_t RestHttpClient::GetRetryDelayMs(int32_t execution_count, const Response* response) const { + return ComputeRetryDelayMs(config_, execution_count, response, + static_cast(std::time(nullptr))); +} + +Result RestHttpClient::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, const std::string& body) const { + if (method != "GET" && method != "POST" && method != "DELETE") { + return Status::Invalid(fmt::format("unsupported http method: {}", method)); + } + std::string url = base_uri_ + path; + if (!query_params.empty()) { + url += "?" + BuildQueryString(query_params); + } + bool idempotent = IsIdempotent(method); + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + int32_t execution_count = 0; + while (true) { + execution_count++; + bool transport_retriable = false; + Result result = ExecuteOnce(method, url, headers, body, &transport_retriable); + bool retriable; + if (result.ok()) { + retriable = IsRetriableCode(result.value().code); + } else { + retriable = idempotent && transport_retriable; + } + if (!retriable || execution_count > config_.max_retries) { + if (result.ok()) { + // Spans all attempts including the backoff sleeps. + int64_t elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + // The request id ties the log line to the server side trace. + auto request_id_iter = result.value().headers.find("x-request-id"); + const std::string request_id = request_id_iter != result.value().headers.end() && + !request_id_iter->second.empty() + ? request_id_iter->second + : "unknown"; + PAIMON_LOG_DEBUG( + logger_, "%s %s finished with http status %lld in %lld ms requestId:%s", + method.c_str(), path.c_str(), + static_cast(result.value().code), // NOLINT(runtime/int) + static_cast(elapsed_ms), // NOLINT(runtime/int) + request_id.c_str()); + } + return result; + } + int64_t delay_ms = + GetRetryDelayMs(execution_count, result.ok() ? &result.value() : nullptr); + std::string reason = result.ok() ? fmt::format("http status {}", result.value().code) + : result.status().ToString(); + PAIMON_LOG_WARN(logger_, "retrying %s %s in %lld ms (retry %d/%d): %s", method.c_str(), + path.c_str(), static_cast(delay_ms), // NOLINT(runtime/int) + execution_count, config_.max_retries, reason.c_str()); + if (delay_ms > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); + } + } +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client.h b/src/paimon/rest/rest_http_client.h new file mode 100644 index 000000000..5d21a087c --- /dev/null +++ b/src/paimon/rest/rest_http_client.h @@ -0,0 +1,125 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/logging.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// HTTP status codes used by the REST catalog. +struct HttpStatus { + static constexpr int64_t kOk = 200; + static constexpr int64_t kAccepted = 202; + static constexpr int64_t kNoContent = 204; + static constexpr int64_t kBadRequest = 400; + static constexpr int64_t kUnauthorized = 401; + static constexpr int64_t kForbidden = 403; + static constexpr int64_t kNotFound = 404; + static constexpr int64_t kConflict = 409; + static constexpr int64_t kTooManyRequests = 429; + static constexpr int64_t kInternalServerError = 500; + static constexpr int64_t kNotImplemented = 501; + static constexpr int64_t kServiceUnavailable = 503; +}; + +/// A blocking HTTP client for the REST catalog based on libcurl. HTTP 429/503 +/// responses are retried for all methods, transport errors only for idempotent +/// methods, with exponential backoff (honoring a `Retry-After` response header, in the +/// delta-seconds or the HTTP-date form, when it yields a positive delay). Redirects to +/// http(s) targets are followed transparently. +class RestHttpClient { + public: + struct Config { + int64_t connect_timeout_ms = 180 * 1000; + int64_t request_timeout_ms = 180 * 1000; + int32_t max_retries = 5; + int64_t retry_base_delay_ms = 1000; + }; + + struct Response { + int64_t code = 0; + std::string body; + /// Response headers with lower-cased names. + std::map headers; + + bool IsSuccessful() const { + return code == HttpStatus::kOk || code == HttpStatus::kAccepted || + code == HttpStatus::kNoContent; + } + }; + + /// Creates a client against `base_uri`. The uri is normalized: surrounding + /// whitespace and all trailing '/' are stripped and "http://" is prepended when no + /// scheme is present. + static Result> Create(const std::string& base_uri); + static Result> Create(const std::string& base_uri, + const Config& config); + + /// Executes `method` ("GET", "POST" or "DELETE") on `path` (already url-encoded, + /// starting with '/'). Query parameter keys and values are url-encoded internally. + /// Returns the final response (which may carry a non-2xx code) or an error status + /// when the request could not be transported at all. Transport errors caused by + /// unresolvable/unreachable hosts, timeouts or TLS failures are never retried. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, + const std::string& body) const; + + const std::string& GetBaseUri() const { + return base_uri_; + } + + static std::string NormalizeUri(const std::string& uri); + + /// Builds "k1=v1&k2=v2" with url-encoded keys and values. + static std::string BuildQueryString(const std::map& query_params); + + /// Computes the delay before the retry following the `execution_count`-th attempt, + /// in ms. A `Retry-After` header of `response` takes precedence when it yields a + /// positive delay (the delta-seconds form, or the HTTP-date form resolved against + /// `now_epoch_seconds`); otherwise the exponential backoff with up to 10% jitter + /// applies. `response` is null when the attempt failed with a transport error. + static int64_t ComputeRetryDelayMs(const Config& config, int32_t execution_count, + const Response* response, int64_t now_epoch_seconds); + + private: + RestHttpClient(const std::string& base_uri, const Config& config); + + /// On a transport failure, `transport_retriable` reports whether the failure kind + /// may be retried (see `Execute` for the non-retriable kinds). + Result ExecuteOnce(const std::string& method, const std::string& url, + const std::map& headers, + const std::string& body, bool* transport_retriable) const; + + /// Backoff before the retry following the `execution_count`-th attempt, in ms. + int64_t GetRetryDelayMs(int32_t execution_count, const Response* response) const; + + /// Keeps libcurl's global state alive for the lifetime of the client. + std::shared_ptr curl_guard_; + std::string base_uri_; + Config config_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp new file mode 100644 index 000000000..aad0eea8d --- /dev/null +++ b/src/paimon/rest/rest_http_client_test.cpp @@ -0,0 +1,421 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { +RestHttpClient::Config FastRetryConfig(int32_t max_retries) { + RestHttpClient::Config config; + config.max_retries = max_retries; + config.retry_base_delay_ms = 1; + return config; +} +} // namespace + +TEST(RestHttpClientTest, NormalizeUri) { + ASSERT_EQ("http://localhost:80", RestHttpClient::NormalizeUri("localhost:80/")); + ASSERT_EQ("http://localhost", RestHttpClient::NormalizeUri("http://localhost//")); + ASSERT_EQ("https://foo.bar", RestHttpClient::NormalizeUri(" https://foo.bar ")); +} + +TEST(RestHttpClientTest, BuildQueryString) { + ASSERT_EQ("a=1&b=x+y%2Fz", RestHttpClient::BuildQueryString({{"a", "1"}, {"b", "x y/z"}})); +} + +TEST(RestHttpClientTest, GetWithHeadersAndQuery) { + // guards last_request: the handler runs on the mock server's accept thread while + // the test thread inspects the captured request + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/config", {{"warehouse", "wh 1"}}, + {{"Authorization", "Bearer token1"}}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ("application/json", response.headers.at("content-type")); + std::lock_guard lock(mutex); + ASSERT_EQ("GET", last_request.method); + ASSERT_EQ("/v1/config", last_request.path); + ASSERT_EQ("wh 1", last_request.query_params.at("warehouse")); + ASSERT_EQ("Bearer token1", last_request.headers.at("authorization")); +} + +TEST(RestHttpClientTest, PostBody) { + // guards last_request: the handler runs on the mock server's accept thread while + // the test thread inspects the captured request + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {{"Content-Type", "application/json"}}, + R"({"name": "db1"})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("POST", last_request.method); + ASSERT_EQ(R"({"name": "db1"})", last_request.body); + ASSERT_EQ("application/json", last_request.headers.at("content-type")); +} + +TEST(RestHttpClientTest, NotFoundIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 404; + response.body = R"({"message": "not found", "code": 404})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(404, response.code); + ASSERT_FALSE(response.IsSuccessful()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ServiceUnavailableIsRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ < 2) { + response.code = 503; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + // 429/503 responses are retried even for non-idempotent POST. + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {}, "{}")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, RetryAfterHeaderIsHonored) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 429; + response.headers["Retry-After"] = "1"; + } + return response; + })); + // the backoff base is large so that the elapsed time below proves the Retry-After + // header took precedence: without it this test would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_GE(elapsed, std::chrono::seconds(1)); + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, ComputeRetryDelayMs) { + RestHttpClient::Config config; + config.retry_base_delay_ms = 1000; + // without a usable Retry-After header: exponential backoff with up to 10% jitter, + // the multiplier capped at 2^6 + int64_t delay = RestHttpClient::ComputeRetryDelayMs(config, 1, nullptr, 0); + ASSERT_GE(delay, 1000); + ASSERT_LE(delay, 1100); + delay = RestHttpClient::ComputeRetryDelayMs(config, 3, nullptr, 0); + ASSERT_GE(delay, 4000); + ASSERT_LE(delay, 4400); + delay = RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0); + ASSERT_GE(delay, 64000); + ASSERT_LE(delay, 70400); + + // the delta-seconds form takes precedence over the backoff, without jitter + RestHttpClient::Response response; + response.headers["retry-after"] = "7"; + ASSERT_EQ(7000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0)); + + // the HTTP-date form is resolved against the given "now" + constexpr int64_t kDateEpoch = 1445412480; // Wed, 21 Oct 2015 07:28:00 GMT + response.headers["retry-after"] = "Wed, 21 Oct 2015 07:28:00 GMT"; + ASSERT_EQ(5000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch - 5)); + + // a date in the past falls back to the backoff + delay = RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch + 1); + ASSERT_GE(delay, 1000); + ASSERT_LE(delay, 1100); +} + +TEST(RestHttpClientTest, InvalidRetryAfterFallsBackToBackoff) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + int32_t count = request_count++; + if (count == 0) { + response.code = 429; + // non-positive values are ignored + response.headers["Retry-After"] = "0"; + } else if (count == 2) { + response.code = 429; + // neither delta-seconds nor a valid http date + response.headers["Retry-After"] = "Sat, 32 Foo 2015 99:99:99 GMT"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_OK_AND_ASSIGN(response, client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(4, request_count.load()); +} + +TEST(RestHttpClientTest, RetriesExhausted) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 503; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(2))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(503, response.code); + // initial attempt + 2 retries + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, EmptyReplyIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + // closing without a response maps to CURLE_GOT_NOTHING, + // one of the non-retriable transport errors (like + // unresolvable hosts, timeouts and TLS failures, which + // cannot be produced deterministically in a unit test) + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ConnectionRefusedIsNotRetried) { + // a closed port yields CURLE_COULDNT_CONNECT, another of the non-retriable + // transport errors; the port of a stopped mock server is known to be closed + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + return MockRestServer::Response(); + })); + std::string base_uri = server->GetBaseUri(); + server->Stop(); + // the backoff base is large so that the elapsed time below proves the failure was + // not retried: a single retry would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(base_uri, config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, DeleteWithBodyIsSent) { + // guards last_request: the handler runs on the mock server's accept thread while + // the test thread inspects the captured request + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + // a body on DELETE keeps the DELETE method line instead of degrading to POST + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {{"Content-Type", "application/json"}}, + R"({"purge": true})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("DELETE", last_request.method); + ASSERT_EQ(R"({"purge": true})", last_request.body); +} + +TEST(RestHttpClientTest, TransportErrorOmitsUrlAndQuery) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + Status status = + client->Execute("GET", "/v1/secret-path", {{"sig", "topsecret1"}}, {}, "").status(); + ASSERT_NOK(status); + // the error must not echo the path or query values, which may carry credentials + // (e.g. a presigned url) + ASSERT_EQ(std::string::npos, status.ToString().find("secret-path")) << status.ToString(); + ASSERT_EQ(std::string::npos, status.ToString().find("topsecret1")) << status.ToString(); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForGet) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = R"({"ok": true})"; + if (request_count++ == 0) { + // cutting the response off mid-body is a retriable + // transport error, and GET is idempotent + response.missing_body_bytes = 5; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForDelete) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + // DELETE is the other idempotent method (see the GET + // test above) + response.missing_body_bytes = 5; + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsNotRetriedForPost) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + // the same retriable transport error kind as in the GET + // test above, but POST is not idempotent + response.missing_body_bytes = 5; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("POST", "/v1/databases", {}, {}, "{}").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, RedirectIsFollowed) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 302; + response.headers["Location"] = "/v1/config"; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/old", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); + // only the final response's headers are reported, not the redirect's + ASSERT_EQ(0, response.headers.count("location")); +} + +TEST(RestHttpClientTest, UnsupportedMethod) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create("http://127.0.0.1:1")); + ASSERT_NOK_WITH_MSG(client->Execute("PATCH", "/", {}, {}, "").status(), + "unsupported http method"); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp new file mode 100644 index 000000000..99ffd056f --- /dev/null +++ b/src/paimon/rest/rest_messages.cpp @@ -0,0 +1,373 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include + +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { + +constexpr const char kFieldMessage[] = "message"; +constexpr const char kFieldResourceType[] = "resourceType"; +constexpr const char kFieldResourceName[] = "resourceName"; +constexpr const char kFieldCode[] = "code"; +constexpr const char kFieldDefaults[] = "defaults"; +constexpr const char kFieldOverrides[] = "overrides"; +constexpr const char kFieldOwner[] = "owner"; +constexpr const char kFieldCreatedAt[] = "createdAt"; +constexpr const char kFieldCreatedBy[] = "createdBy"; +constexpr const char kFieldUpdatedAt[] = "updatedAt"; +constexpr const char kFieldUpdatedBy[] = "updatedBy"; +constexpr const char kFieldName[] = "name"; +constexpr const char kFieldOptions[] = "options"; +constexpr const char kFieldId[] = "id"; +constexpr const char kFieldLocation[] = "location"; +constexpr const char kFieldDatabases[] = "databases"; +constexpr const char kFieldTables[] = "tables"; +constexpr const char kFieldSnapshots[] = "snapshots"; +constexpr const char kFieldNextPageToken[] = "nextPageToken"; +constexpr const char kFieldPath[] = "path"; +constexpr const char kFieldIsExternal[] = "isExternal"; +constexpr const char kFieldSchemaId[] = "schemaId"; +constexpr const char kFieldSchema[] = "schema"; +constexpr const char kFieldIdentifier[] = "identifier"; +constexpr const char kFieldDatabase[] = "database"; +constexpr const char kFieldObject[] = "object"; +constexpr const char kFieldSource[] = "source"; +constexpr const char kFieldDestination[] = "destination"; + +void AddOptionalStringMember(rapidjson::Value* obj, const char* key, + const std::optional& value, + rapidjson::Document::AllocatorType* allocator) { + if (value) { + obj->AddMember(rapidjson::StringRef(key), + RapidJsonUtil::SerializeValue(value.value(), allocator).Move(), *allocator); + } +} + +rapidjson::Value SerializeIdentifier(const std::string& database, const std::string& table, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabase), + RapidJsonUtil::SerializeValue(database, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldObject), + RapidJsonUtil::SerializeValue(table, allocator).Move(), *allocator); + return obj; +} + +void DeserializeIdentifier(const rapidjson::Value& obj, const char* key, std::string* database, + std::string* table) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + const rapidjson::Value& identifier = obj[key]; + *database = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldDatabase); + *table = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldObject); +} + +std::string DeserializeRawJsonMember(const rapidjson::Value& obj, const char* key) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + return RestUtil::JsonToString(obj[key]); +} + +} // namespace + +rapidjson::Value ErrorResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldMessage), + RapidJsonUtil::SerializeValue(message_, allocator).Move(), *allocator); + if (!resource_type_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceType), + RapidJsonUtil::SerializeValue(resource_type_, allocator).Move(), *allocator); + } + if (!resource_name_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceName), + RapidJsonUtil::SerializeValue(resource_name_, allocator).Move(), *allocator); + } + obj.AddMember(rapidjson::StringRef(kFieldCode), + RapidJsonUtil::SerializeValue(code_, allocator).Move(), *allocator); + return obj; +} + +void ErrorResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + message_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldMessage, std::string()); + resource_type_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceType, std::string()); + resource_name_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceName, std::string()); + code_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldCode, 0); +} + +rapidjson::Value ConfigResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDefaults), + RapidJsonUtil::SerializeValue(defaults_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOverrides), + RapidJsonUtil::SerializeValue(overrides_, allocator).Move(), *allocator); + return obj; +} + +namespace { +/// The server may send null values in `defaults`/`overrides`; merging filters them +/// out, so they are skipped here instead of failing the whole parse. +std::map DeserializeStringMapSkippingNulls(const rapidjson::Value& obj, + const char* key) { + std::map result; + if (!obj.IsObject() || !obj.HasMember(key) || obj[key].IsNull()) { + return result; + } + const rapidjson::Value& map_value = obj[key]; + if (!map_value.IsObject()) { + throw std::invalid_argument(std::string("member '") + key + "' must be an object"); + } + for (auto iter = map_value.MemberBegin(); iter != map_value.MemberEnd(); ++iter) { + if (iter->value.IsNull()) { + continue; + } + if (!iter->value.IsString()) { + throw std::invalid_argument(std::string("member '") + key + + "' must only contain string values"); + } + result[iter->name.GetString()] = iter->value.GetString(); + } + return result; +} +} // namespace + +void ConfigResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + defaults_ = DeserializeStringMapSkippingNulls(obj, kFieldDefaults); + overrides_ = DeserializeStringMapSkippingNulls(obj, kFieldOverrides); +} + +std::map ConfigResponse::Merge( + const std::map& client_options) const { + std::map merged = defaults_; + for (const auto& [key, value] : client_options) { + merged[key] = value; + } + for (const auto& [key, value] : overrides_) { + merged[key] = value; + } + return merged; +} + +void RestAuditFields::ParseFrom(const rapidjson::Value& obj) { + owner = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldOwner, + std::nullopt); + created_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldCreatedAt, + std::nullopt); + created_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldCreatedBy, std::nullopt); + updated_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldUpdatedAt, + std::nullopt); + updated_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldUpdatedBy, std::nullopt); +} + +void RestAuditFields::AddTo(rapidjson::Value* obj, + rapidjson::Document::AllocatorType* allocator) const { + AddOptionalStringMember(obj, kFieldOwner, owner, allocator); + if (created_at) { + obj->AddMember(rapidjson::StringRef(kFieldCreatedAt), + RapidJsonUtil::SerializeValue(created_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldCreatedBy, created_by, allocator); + if (updated_at) { + obj->AddMember(rapidjson::StringRef(kFieldUpdatedAt), + RapidJsonUtil::SerializeValue(updated_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldUpdatedBy, updated_by, allocator); +} + +void RestAuditFields::PutAuditOptionsTo(std::map* options) const { + if (owner) { + (*options)[kFieldOwner] = owner.value(); + } + if (created_at) { + (*options)[kFieldCreatedAt] = std::to_string(created_at.value()); + } + if (created_by) { + (*options)[kFieldCreatedBy] = created_by.value(); + } + if (updated_at) { + (*options)[kFieldUpdatedAt] = std::to_string(updated_at.value()); + } + if (updated_by) { + (*options)[kFieldUpdatedBy] = updated_by.value(); + } +} + +rapidjson::Value CreateDatabaseRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + return obj; +} + +void CreateDatabaseRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); +} + +rapidjson::Value GetDatabaseResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldLocation), + RapidJsonUtil::SerializeValue(location_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetDatabaseResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + location_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldLocation, std::string()); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); + audit_.ParseFrom(obj); +} + +rapidjson::Value ListDatabasesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabases), + RapidJsonUtil::SerializeValue(databases_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListDatabasesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + databases_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldDatabases, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListTablesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldTables), + RapidJsonUtil::SerializeValue(tables_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListTablesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + tables_ = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldTables, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListSnapshotsResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSnapshots), + RapidJsonUtil::SerializeValue(snapshots_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListSnapshotsResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + snapshots_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldSnapshots, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value GetTableResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldPath), + RapidJsonUtil::SerializeValue(path_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldIsExternal), + RapidJsonUtil::SerializeValue(is_external_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchemaId), + RapidJsonUtil::SerializeValue(schema_id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetTableResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName, std::string()); + path_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldPath); + is_external_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldIsExternal, false); + schema_id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldSchemaId); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); + audit_.ParseFrom(obj); +} + +rapidjson::Value CreateTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldIdentifier), + SerializeIdentifier(database_, table_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + return obj; +} + +void CreateTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldIdentifier, &database_, &table_); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); +} + +rapidjson::Value RenameTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSource), + SerializeIdentifier(source_database_, source_table_, allocator).Move(), + *allocator); + obj.AddMember(rapidjson::StringRef(kFieldDestination), + SerializeIdentifier(destination_database_, destination_table_, allocator).Move(), + *allocator); + return obj; +} + +void RenameTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldSource, &source_database_, &source_table_); + DeserializeIdentifier(obj, kFieldDestination, &destination_database_, &destination_table_); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h new file mode 100644 index 000000000..d05b34c1f --- /dev/null +++ b/src/paimon/rest/rest_messages.h @@ -0,0 +1,374 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/common/utils/jsonizable.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/snapshot.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +/// Request and response objects of the REST catalog protocol. The JSON field names +/// must stay aligned with the REST catalog open api. + +class ErrorResponse : public Jsonizable { + public: + static constexpr const char* kResourceTypeDatabase = "DATABASE"; + static constexpr const char* kResourceTypeTable = "TABLE"; + + ErrorResponse(const std::string& resource_type, const std::string& resource_name, + const std::string& message, int32_t code) + : resource_type_(resource_type), + resource_name_(resource_name), + message_(message), + code_(code) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetResourceType() const { + return resource_type_; + } + const std::string& GetResourceName() const { + return resource_name_; + } + const std::string& GetMessage() const { + return message_; + } + int32_t GetCode() const { + return code_; + } + + ErrorResponse() = default; + + private: + std::string resource_type_; + std::string resource_name_; + std::string message_; + int32_t code_ = 0; +}; + +/// The response of "/v1/config". +class ConfigResponse : public Jsonizable { + public: + ConfigResponse(const std::map& defaults, + const std::map& overrides) + : defaults_(defaults), overrides_(overrides) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + /// Merges with the client options; the precedence is + /// `overrides` > `client_options` > `defaults`. + std::map Merge( + const std::map& client_options) const; + + const std::map& GetDefaults() const { + return defaults_; + } + const std::map& GetOverrides() const { + return overrides_; + } + + ConfigResponse() = default; + + private: + std::map defaults_; + std::map overrides_; +}; + +/// The audit fields shared by database/table responses. +struct RestAuditFields { + std::optional owner; + std::optional created_at; + std::optional created_by; + std::optional updated_at; + std::optional updated_by; + + void ParseFrom(const rapidjson::Value& obj); + void AddTo(rapidjson::Value* obj, rapidjson::Document::AllocatorType* allocator) const; + /// Adds the present fields to `options`. + void PutAuditOptionsTo(std::map* options) const; +}; + +class CreateDatabaseRequest : public Jsonizable { + public: + CreateDatabaseRequest(const std::string& name, + const std::map& options) + : name_(name), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetName() const { + return name_; + } + const std::map& GetOptions() const { + return options_; + } + + CreateDatabaseRequest() = default; + + private: + std::string name_; + std::map options_; +}; + +class GetDatabaseResponse : public Jsonizable { + public: + GetDatabaseResponse(const std::string& id, const std::string& name, const std::string& location, + const std::map& options) + : id_(id), name_(name), location_(location), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetLocation() const { + return location_; + } + const std::map& GetOptions() const { + return options_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetDatabaseResponse() = default; + + private: + std::string id_; + std::string name_; + std::string location_; + std::map options_; + RestAuditFields audit_; +}; + +class ListDatabasesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListDatabasesResponse(const std::vector& databases, + const std::optional& next_page_token) + : databases_(databases), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return databases_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListDatabasesResponse() = default; + + private: + std::vector databases_; + std::optional next_page_token_; +}; + +class ListTablesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListTablesResponse(const std::vector& tables, + const std::optional& next_page_token) + : tables_(tables), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return tables_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListTablesResponse() = default; + + private: + std::vector tables_; + std::optional next_page_token_; +}; + +/// Each element is a snapshot in the same JSON layout as the snapshot files. +class ListSnapshotsResponse : public Jsonizable { + public: + using ItemType = Snapshot; + + ListSnapshotsResponse(const std::vector& snapshots, + const std::optional& next_page_token) + : snapshots_(snapshots), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return snapshots_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListSnapshotsResponse() = default; + + private: + std::vector snapshots_; + std::optional next_page_token_; +}; + +/// The nested `schema` object (fields/partitionKeys/primaryKeys/options/comment) is +/// kept as a raw JSON string and converted to a `TableSchema` by the catalog. +class GetTableResponse : public Jsonizable { + public: + GetTableResponse(const std::string& id, const std::string& name, const std::string& path, + bool is_external, int64_t schema_id, const std::string& schema_json) + : id_(id), + name_(name), + path_(path), + is_external_(is_external), + schema_id_(schema_id), + schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetPath() const { + return path_; + } + bool IsExternal() const { + return is_external_; + } + int64_t GetSchemaId() const { + return schema_id_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetTableResponse() = default; + + private: + std::string id_; + std::string name_; + std::string path_; + bool is_external_ = false; + int64_t schema_id_ = 0; + std::string schema_json_; + RestAuditFields audit_; +}; + +/// `schema_json` uses the same schema JSON layout as `GetTableResponse`. +class CreateTableRequest : public Jsonizable { + public: + CreateTableRequest(const std::string& database, const std::string& table, + const std::string& schema_json) + : database_(database), table_(table), schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetDatabase() const { + return database_; + } + const std::string& GetTable() const { + return table_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + + CreateTableRequest() = default; + + private: + std::string database_; + std::string table_; + std::string schema_json_; +}; + +class RenameTableRequest : public Jsonizable { + public: + RenameTableRequest(const std::string& source_database, const std::string& source_table, + const std::string& destination_database, + const std::string& destination_table) + : source_database_(source_database), + source_table_(source_table), + destination_database_(destination_database), + destination_table_(destination_table) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetSourceDatabase() const { + return source_database_; + } + const std::string& GetSourceTable() const { + return source_table_; + } + const std::string& GetDestinationDatabase() const { + return destination_database_; + } + const std::string& GetDestinationTable() const { + return destination_table_; + } + + RenameTableRequest() = default; + + private: + std::string source_database_; + std::string source_table_; + std::string destination_database_; + std::string destination_table_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp new file mode 100644 index 000000000..b75796227 --- /dev/null +++ b/src/paimon/rest/rest_messages_test.cpp @@ -0,0 +1,293 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/rest/resource_paths.h" +#include "paimon/rest/rest_util.h" +#include "paimon/testing/utils/testharness.h" +#include "rapidjson/document.h" + +namespace paimon::test { + +TEST(RestUtilTest, ExtractPrefixMap) { + std::map options = { + {"header.k1", "v1"}, {"header.k2", "v2"}, {"other", "v3"}, {"header.", "v4"}}; + std::map expected = {{"k1", "v1"}, {"k2", "v2"}}; + ASSERT_EQ(expected, RestUtil::ExtractPrefixMap(options, "header.")); +} + +TEST(RestUtilTest, RedactText) { + // markers are matched on the text normalized to lower-case alphanumerics, so + // separators cannot be used to smuggle a secret through + ASSERT_EQ("******", RestUtil::RedactText("invalid password=abc123")); + ASSERT_EQ("******", RestUtil::RedactText("bad ACCESS-KEY provided")); + ASSERT_EQ("******", RestUtil::RedactText("url?sig=deadbeef")); + ASSERT_EQ("table t1 not found", RestUtil::RedactText("table t1 not found")); + ASSERT_EQ("", RestUtil::RedactText("")); +} + +TEST(ResourcePathsTest, WithPrefix) { + ResourcePaths paths("my prefix"); + ASSERT_EQ("/v1/config", ResourcePaths::Config()); + ASSERT_EQ("/v1/my+prefix/databases", paths.Databases()); + ASSERT_EQ("/v1/my+prefix/databases/db%231", paths.Database("db#1")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables", paths.Tables("db")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1", paths.Table("db", "t1")); + ASSERT_EQ("/v1/my+prefix/tables/rename", paths.RenameTable()); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/snapshots", paths.Snapshots("db", "t1")); +} + +TEST(ResourcePathsTest, WithoutPrefix) { + ResourcePaths paths(""); + ASSERT_EQ("/v1/databases", paths.Databases()); + ASSERT_EQ("/v1/tables/rename", paths.RenameTable()); +} + +TEST(RestMessagesTest, ErrorResponseRoundTrip) { + ErrorResponse response(ErrorResponse::kResourceTypeDatabase, "db1", "database db1 not found", + 404); + ASSERT_OK_AND_ASSIGN(std::string json, response.ToJsonString()); + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString(json)); + ASSERT_EQ("DATABASE", parsed.GetResourceType()); + ASSERT_EQ("db1", parsed.GetResourceName()); + ASSERT_EQ("database db1 not found", parsed.GetMessage()); + ASSERT_EQ(404, parsed.GetCode()); +} + +TEST(RestMessagesTest, ErrorResponseLenientParse) { + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString("{}")); + ASSERT_EQ("", parsed.GetMessage()); + ASSERT_EQ(0, parsed.GetCode()); +} + +TEST(RestMessagesTest, ConfigResponseMerge) { + // null values may be sent by the server and are filtered out + std::string json = R"({ + "defaults": {"prefix": "server-prefix", "a": "default-a", "b": "default-b", + "nullable": null}, + "overrides": {"c": "override-c", "a": "override-a"} + })"; + ASSERT_OK_AND_ASSIGN(ConfigResponse config, ConfigResponse::FromJsonString(json)); + std::map client = { + {"a", "client-a"}, {"b", "client-b"}, {"d", "client-d"}}; + std::map merged = config.Merge(client); + // overrides > client options > defaults + ASSERT_EQ("override-a", merged["a"]); + ASSERT_EQ("client-b", merged["b"]); + ASSERT_EQ("override-c", merged["c"]); + ASSERT_EQ("client-d", merged["d"]); + ASSERT_EQ("server-prefix", merged["prefix"]); + ASSERT_EQ(0, merged.count("nullable")); +} + +TEST(RestMessagesTest, ListResponsesParse) { + ASSERT_OK_AND_ASSIGN(ListDatabasesResponse databases, + ListDatabasesResponse::FromJsonString( + R"({"databases": ["db1", "db2"], "nextPageToken": "token1"})")); + ASSERT_EQ((std::vector{"db1", "db2"}), databases.Data()); + ASSERT_EQ("token1", databases.NextPageToken().value()); + + ASSERT_OK_AND_ASSIGN(ListTablesResponse tables, + ListTablesResponse::FromJsonString(R"({"tables": ["t1"]})")); + ASSERT_EQ((std::vector{"t1"}), tables.Data()); + ASSERT_FALSE(tables.NextPageToken().has_value()); + + // nextPageToken may be serialized as explicit null by the server + ASSERT_OK_AND_ASSIGN( + ListTablesResponse null_token, + ListTablesResponse::FromJsonString(R"({"tables": [], "nextPageToken": null})")); + ASSERT_FALSE(null_token.NextPageToken().has_value()); +} + +TEST(RestMessagesTest, ListSnapshotsResponseParse) { + std::string json = R"({ + "snapshots": [{ + "version": 3, + "id": 7, + "schemaId": 2, + "baseManifestList": "manifest-list-1", + "deltaManifestList": "manifest-list-2", + "commitUser": "user1", + "commitIdentifier": 9, + "commitKind": "APPEND", + "timeMillis": 1234567, + "totalRecordCount": 100, + "deltaRecordCount": 10, + "watermark": 42 + }, { + "version": 3, + "id": 8, + "schemaId": 2, + "baseManifestList": "manifest-list-3", + "deltaManifestList": "manifest-list-4", + "commitUser": "user1", + "commitIdentifier": 10, + "commitKind": "COMPACT", + "timeMillis": 1234568, + "totalRecordCount": 100, + "deltaRecordCount": 0 + }], + "nextPageToken": null + })"; + ASSERT_OK_AND_ASSIGN(ListSnapshotsResponse response, + ListSnapshotsResponse::FromJsonString(json)); + ASSERT_EQ(2, response.Data().size()); + const Snapshot& snapshot = response.Data()[0]; + ASSERT_EQ(7, snapshot.Id()); + ASSERT_EQ(2, snapshot.SchemaId()); + ASSERT_EQ("user1", snapshot.CommitUser()); + ASSERT_EQ(1234567, snapshot.TimeMillis()); + SnapshotInfo info = snapshot.ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(100, info.total_record_count.value()); + ASSERT_EQ(42, info.watermark.value()); + SnapshotInfo compact_info = response.Data()[1].ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, compact_info.commit_kind); + ASSERT_FALSE(compact_info.watermark.has_value()); +} + +TEST(RestMessagesTest, GetDatabaseResponseParse) { + std::string json = R"({ + "id": "10", + "name": "db1", + "location": "/warehouse/db1.db", + "options": {"k1": "v1"}, + "owner": "owner1", + "createdAt": 100, + "createdBy": "creator", + "updatedAt": 200, + "updatedBy": "updater" + })"; + ASSERT_OK_AND_ASSIGN(GetDatabaseResponse response, GetDatabaseResponse::FromJsonString(json)); + ASSERT_EQ("db1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db", response.GetLocation()); + ASSERT_EQ("v1", response.GetOptions().at("k1")); + std::map options; + response.GetAuditFields().PutAuditOptionsTo(&options); + ASSERT_EQ("owner1", options.at("owner")); + ASSERT_EQ("100", options.at("createdAt")); + ASSERT_EQ("updater", options.at("updatedBy")); +} + +TEST(RestMessagesTest, GetTableResponseParse) { + std::string json = R"({ + "id": "42", + "name": "t1", + "path": "/warehouse/db1.db/t1", + "isExternal": false, + "schemaId": 3, + "schema": { + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "f1", "type": "STRING"} + ], + "partitionKeys": [], + "primaryKeys": ["f0"], + "options": {"bucket": "2"}, + "comment": "a table" + }, + "updatedAt": 300 + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("42", response.GetId()); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db/t1", response.GetPath()); + ASSERT_FALSE(response.IsExternal()); + ASSERT_EQ(3, response.GetSchemaId()); + ASSERT_EQ(300, response.GetAuditFields().updated_at.value()); + rapidjson::Document schema; + schema.Parse(response.GetSchemaJson().c_str()); + ASSERT_FALSE(schema.HasParseError()); + ASSERT_EQ(2u, schema["fields"].Size()); + ASSERT_STREQ("f0", schema["fields"][0]["name"].GetString()); +} + +TEST(RestMessagesTest, UnknownFieldsAreIgnored) { + // forward compatibility with newer servers: fields the client does not know must + // be ignored + std::string json = R"({ + "id": "42", "name": "t1", "path": "p", "schemaId": 3, + "schema": {"fields": []}, + "futureField": {"nested": true}, "anotherUnknown": [1, 2] + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ(3, response.GetSchemaId()); +} + +TEST(RestMessagesTest, MissingRequiredFieldsFail) { + // "path", "schemaId" and "schema" are required + ASSERT_NOK(GetTableResponse::FromJsonString(R"({"id": "42", "name": "t1"})").status()); + // a non-object identifier is rejected + CreateTableRequest bad_identifier("", "", ""); + ASSERT_NOK(RapidJsonUtil::FromJsonString(R"({"identifier": "not-an-object", "schema": {}})", + &bad_identifier)); +} + +TEST(RestMessagesTest, CreateTableRequestSerialize) { + CreateTableRequest request( + "db1", "t1", R"({"fields": [], "partitionKeys": [], "primaryKeys": [], "options": {}})"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("db1", doc["identifier"]["database"].GetString()); + ASSERT_STREQ("t1", doc["identifier"]["object"].GetString()); + ASSERT_TRUE(doc["schema"].IsObject()); + ASSERT_TRUE(doc["schema"]["fields"].IsArray()); + + ASSERT_OK_AND_ASSIGN(CreateTableRequest parsed, CreateTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetDatabase()); + ASSERT_EQ("t1", parsed.GetTable()); +} + +TEST(RestMessagesTest, InvalidSchemaJsonErrorOmitsPayload) { + CreateTableRequest request("db1", "t1", R"({"fields": [], "options": {"token": "top-secret")"); + Status status = request.ToJsonString().status(); + ASSERT_NOK_WITH_MSG(status, "invalid json"); + // the payload may carry credentials, so it must not be echoed back in the error + ASSERT_EQ(std::string::npos, status.ToString().find("top-secret")) << status.ToString(); +} + +TEST(RestMessagesTest, RenameTableRequestSerialize) { + RenameTableRequest request("db1", "t1", "db1", "t2"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("t1", doc["source"]["object"].GetString()); + ASSERT_STREQ("t2", doc["destination"]["object"].GetString()); + + ASSERT_OK_AND_ASSIGN(RenameTableRequest parsed, RenameTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetSourceDatabase()); + ASSERT_EQ("t2", parsed.GetDestinationTable()); +} + +TEST(RestMessagesTest, CreateDatabaseRequestRoundTrip) { + CreateDatabaseRequest request("db1", {{"k1", "v1"}}); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + ASSERT_OK_AND_ASSIGN(CreateDatabaseRequest parsed, CreateDatabaseRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetName()); + ASSERT_EQ("v1", parsed.GetOptions().at("k1")); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_util.cpp b/src/paimon/rest/rest_util.cpp new file mode 100644 index 000000000..ffb5904a0 --- /dev/null +++ b/src/paimon/rest/rest_util.cpp @@ -0,0 +1,92 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_util.h" + +#include + +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "rapidjson/error/en.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { + +std::map RestUtil::ExtractPrefixMap( + const std::map& options, const std::string& prefix) { + std::map result; + for (const auto& [key, value] : options) { + if (key.size() > prefix.size() && key.compare(0, prefix.size(), prefix) == 0) { + result[key.substr(prefix.size())] = value; + } + } + return result; +} + +std::string RestUtil::RedactText(const std::string& text) { + if (text.empty()) { + return text; + } + static constexpr const char kRedacted[] = "******"; + std::string lower = StringUtils::ToLowerCase(text); + static constexpr const char* kLiteralMarkers[] = {"sig=", "\"sig\"", "'sig'"}; + for (const char* marker : kLiteralMarkers) { + if (lower.find(marker) != std::string::npos) { + return kRedacted; + } + } + std::string normalized; + normalized.reserve(lower.size()); + for (char c : lower) { + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) { + normalized.push_back(c); + } + } + static constexpr const char* kMarkers[] = { + "password", "secret", "token", "credential", "accesskey", "accountkey", + "encryptionkey", "authorization", "privatekey", "apikey", "signature", "sas"}; + for (const char* marker : kMarkers) { + if (normalized.find(marker) != std::string::npos) { + return kRedacted; + } + } + return text; +} + +std::string RestUtil::JsonToString(const rapidjson::Value& value) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return buffer.GetString(); +} + +rapidjson::Value RestUtil::ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + // The payload may carry credentials and be arbitrarily large; report only the error. + throw std::invalid_argument(fmt::format("invalid json: {} (at offset {})", + rapidjson::GetParseError_En(doc.GetParseError()), + doc.GetErrorOffset())); + } + rapidjson::Value value; + value.CopyFrom(doc, *allocator); + return value; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_util.h b/src/paimon/rest/rest_util.h new file mode 100644 index 000000000..347dc4bd8 --- /dev/null +++ b/src/paimon/rest/rest_util.h @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +/// Utilities for the REST catalog; URL encoding and decoding live in `UrlUtils`. +class RestUtil { + public: + RestUtil() = delete; + ~RestUtil() = delete; + + /// Extract all options whose key starts with `prefix`, with the prefix stripped from + /// the resulting keys. A key exactly equal to the prefix is dropped instead of + /// yielding an empty key. + static std::map ExtractPrefixMap( + const std::map& options, const std::string& prefix); + + /// Redacts free-form text such as a server error message: when any marker of a + /// sensitive value ("password", "token", "sig=", ...) is present the whole text is + /// replaced by "******", since arbitrary text cannot be masked per-secret reliably. + static std::string RedactText(const std::string& text); + + /// Serialize a rapidjson value to a compact JSON string. + static std::string JsonToString(const rapidjson::Value& value); + + /// Parse a JSON string into a rapidjson value owned by `allocator`. + /// Throws `std::invalid_argument` on parse error, consistent with `Jsonizable`. + static rapidjson::Value ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator); +}; + +} // namespace paimon