From 4096f12a4f4d7344abf7707a4967514a0d950159 Mon Sep 17 00:00:00 2001 From: Zee Yin Date: Tue, 12 May 2026 17:15:25 +0800 Subject: [PATCH 01/13] feat: add support for CRC32C checksum in S3 uploads and disable Content-MD5 for S3 Express --- be/src/common/config.cpp | 2 + be/src/common/config.h | 6 +++ be/src/io/fs/s3_obj_storage_client.cpp | 51 ++++++++++++++++++++++++-- be/src/io/fs/s3_obj_storage_client.h | 7 +++- be/src/util/s3_util.cpp | 3 +- 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 9ec587bd100ec5..5fa1fa6091973d 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1512,6 +1512,8 @@ DEFINE_mInt32(s3_read_max_wait_time_ms, "800"); DEFINE_mBool(enable_s3_object_check_after_upload, "true"); DEFINE_mInt32(aws_client_request_timeout_ms, "30000"); +DEFINE_mBool(s3_disable_content_md5, "false"); + DEFINE_mBool(enable_s3_rate_limiter, "false"); DEFINE_mInt64(s3_get_bucket_tokens, "1000000000000000000"); DEFINE_Validator(s3_get_bucket_tokens, [](int64_t config) -> bool { return config > 0; }); diff --git a/be/src/common/config.h b/be/src/common/config.h index 012f09a735c0cd..c9a6241b45882f 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1621,6 +1621,12 @@ DECLARE_mInt32(s3_read_max_wait_time_ms); DECLARE_mBool(enable_s3_object_check_after_upload); DECLARE_mInt32(aws_client_request_timeout_ms); +// When true, omit the Content-MD5 header on S3 PutObject / UploadPart and send a +// CRC32C checksum instead. Required for S3 Express One Zone, which returns 501 +// NotImplemented for Content-MD5. Endpoints containing "s3express" auto-enable +// this regardless of the flag. +DECLARE_mBool(s3_disable_content_md5); + // write as inverted index tmp directory DECLARE_String(tmp_file_dir); diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 1b8bf7b473c076..3950f23bb034a9 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -61,9 +62,12 @@ #include #include +#include + #include #include +#include "common/config.h" #include "common/logging.h" #include "common/status.h" #include "cpp/sync_point.h" @@ -116,6 +120,31 @@ using namespace Aws::S3::Model; static constexpr int S3_REQUEST_THRESHOLD_MS = 5000; +namespace { +// S3 Express One Zone endpoints follow the pattern +// "*.s3express-..amazonaws.com" — substring match is sufficient +// for both the gateway and the s3express subdomain forms. +bool is_s3_express_endpoint(const std::string& endpoint) { + return endpoint.find("s3express") != std::string::npos; +} + +// AWS expects the CRC32C value as the big-endian 4-byte representation, base64-encoded. +Aws::String compute_crc32c_b64(std::string_view data) { + uint32_t crc = crc32c::Crc32c(reinterpret_cast(data.data()), data.size()); + uint8_t bytes[4] = {static_cast((crc >> 24) & 0xff), + static_cast((crc >> 16) & 0xff), + static_cast((crc >> 8) & 0xff), + static_cast(crc & 0xff)}; + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::ByteBuffer(bytes, sizeof(bytes))); +} +} // namespace + +S3ObjStorageClient::S3ObjStorageClient(std::shared_ptr client, + const std::string& endpoint) + : _client(std::move(client)), + _disable_content_md5(config::s3_disable_content_md5 || + is_s3_express_endpoint(endpoint)) {} + ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { CreateMultipartUploadRequest request; @@ -158,8 +187,15 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti Aws::S3::Model::PutObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); auto string_view_stream = std::make_shared(stream.data(), stream.size()); - Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + if (_disable_content_md5) { + // S3 Express One Zone rejects Content-MD5; use CRC32C instead. + request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); + request.SetChecksumCRC32C(compute_crc32c_b64(stream)); + } else { + Aws::Utils::ByteBuffer part_md5( + Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + } request.SetBody(string_view_stream); request.SetContentLength(stream.size()); request.SetContentType("application/octet-stream"); @@ -202,8 +238,15 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP request.SetBody(string_view_stream); - Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + if (_disable_content_md5) { + // S3 Express One Zone rejects Content-MD5; use CRC32C instead. + request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); + request.SetChecksumCRC32C(compute_crc32c_b64(stream)); + } else { + Aws::Utils::ByteBuffer part_md5( + Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + } request.SetContentLength(stream.size()); request.SetContentType("application/octet-stream"); diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 45294226594d81..ac048a65a98238 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -32,7 +32,8 @@ class ObjClientHolder; class S3ObjStorageClient final : public ObjStorageClient { public: - S3ObjStorageClient(std::shared_ptr client) : _client(std::move(client)) {} + explicit S3ObjStorageClient(std::shared_ptr client, + const std::string& endpoint = {}); ~S3ObjStorageClient() override = default; ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override; @@ -58,6 +59,10 @@ class S3ObjStorageClient final : public ObjStorageClient { private: std::shared_ptr _client; + // True for S3 Express One Zone endpoints (or when config::s3_disable_content_md5 + // is on). When set, uploads send a CRC32C checksum instead of Content-MD5, + // since S3 Express returns 501 NotImplemented for the latter. + bool _disable_content_md5 = false; }; } // namespace doris::io diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index fc4c81d4bd96f2..6218c524362fe8 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -532,7 +532,8 @@ std::shared_ptr S3ClientFactory::_create_s3_client( Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, s3_conf.use_virtual_addressing); - auto obj_client = std::make_shared(std::move(new_client)); + auto obj_client = + std::make_shared(std::move(new_client), s3_conf.endpoint); LOG_INFO("create one s3 client with {}", s3_conf.to_string()); return obj_client; } From 98b7adffee8639c42fc448a30d3ae49aa46f249a Mon Sep 17 00:00:00 2001 From: Zee Yin Date: Thu, 14 May 2026 14:21:04 +0800 Subject: [PATCH 02/13] Upgrade AWS SDK and use new create client to support S3 express --- be/src/util/s3_util.cpp | 28 ++++++++++++++++++++-------- common/cpp/aws_logger.h | 9 +++++++++ thirdparty/vars.sh | 8 ++++---- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 6218c524362fe8..3fd802f5b8fe9d 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -491,11 +491,13 @@ std::shared_ptr S3ClientFactory::_create_s3_client( TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", std::make_shared(std::make_shared())); - Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); - if (s3_conf.need_override_endpoint) { - aws_config.endpointOverride = s3_conf.endpoint; - } + // Use S3ClientConfiguration (not the legacy ClientConfiguration) so the SDK's + // endpoint-rules resolver is active. This is required for S3 Express One Zone: + // the resolver detects the --x-s3 bucket suffix, calls CreateSession, and signs + // requests with service name "s3express" instead of "s3". + Aws::S3::S3ClientConfiguration aws_config; aws_config.region = s3_conf.region; + aws_config.useVirtualAddressing = s3_conf.use_virtual_addressing; if (_ca_cert_file_path.empty()) { _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); @@ -527,10 +529,20 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.retryStrategy = std::make_shared( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/false); - std::shared_ptr new_client = std::make_shared( - get_aws_credentials_provider(s3_conf), std::move(aws_config), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - s3_conf.use_virtual_addressing); + // S3 Express buckets are identified by the --x-s3 suffix or an s3express endpoint. + // Skip endpointOverride for these so the SDK resolves the bucket-specific endpoint + // and manages CreateSession automatically. For all other buckets, keep the override. + const bool is_s3_express = s3_conf.endpoint.find("s3express") != std::string::npos || + s3_conf.bucket.find("--x-s3") != std::string::npos; + if (s3_conf.need_override_endpoint && !is_s3_express) { + aws_config.endpointOverride = s3_conf.endpoint; + } + aws_config.disableS3ExpressAuth = !is_s3_express; + + auto new_client = std::make_shared( + get_aws_credentials_provider(s3_conf), + Aws::MakeShared("S3Client"), + aws_config); auto obj_client = std::make_shared(std::move(new_client), s3_conf.endpoint); diff --git a/common/cpp/aws_logger.h b/common/cpp/aws_logger.h index 85734d13e1411c..9f06cec80fac0f 100644 --- a/common/cpp/aws_logger.h +++ b/common/cpp/aws_logger.h @@ -20,6 +20,8 @@ #include #include #include +#include +#include class DorisAWSLogger final : public Aws::Utils::Logging::LogSystemInterface { public: @@ -36,6 +38,13 @@ class DorisAWSLogger final : public Aws::Utils::Logging::LogSystemInterface { _log_impl(log_level, tag, message_stream.str().c_str()); } + void vaLog(Aws::Utils::Logging::LogLevel log_level, const char* tag, const char* format_str, + va_list args) final { + char buf[4096]; + vsnprintf(buf, sizeof(buf), format_str, args); + _log_impl(log_level, tag, buf); + } + void Flush() final {} private: diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index af46e566b8a30f..1b4ca87919f81b 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -356,10 +356,10 @@ BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css" BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a" # aws sdk -AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.219.tar.gz" -AWS_SDK_NAME="aws-sdk-cpp-1.11.219.tar.gz" -AWS_SDK_SOURCE="aws-sdk-cpp-1.11.219" -AWS_SDK_MD5SUM="80aa616efe1a3e7a9bf0dfbc44a97864" +AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.400.tar.gz" +AWS_SDK_NAME="aws-sdk-cpp-1.11.400.tar.gz" +AWS_SDK_SOURCE="aws-sdk-cpp-1.11.400" +AWS_SDK_MD5SUM="329c46612df55ba6715d9d2ccd61dc03" # tsan_header TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7" From 0a193178c855d220f0817e26de0efd5003be6a4d Mon Sep 17 00:00:00 2001 From: Refrain Date: Mon, 13 Jul 2026 09:26:42 +0800 Subject: [PATCH 03/13] [fix](be) Detect S3 Express context accurately ### What problem does this PR solve?\n\nIssue Number: close #xxx\n\nRelated PR: #63409\n\nProblem Summary: The initial S3 Express implementation detected directory buckets and endpoints with broad substring matches in two separate locations. This could classify ordinary bucket names or endpoint paths as S3 Express and allowed the client configuration and upload checksum behavior to diverge. Centralize the decision in one helper, require the documented --x-s3 bucket suffix or an s3express endpoint host label, and pass the resulting context into the object storage client. Add unit coverage for valid and invalid names.\n\n### Release note\n\nNone\n\n### Check List (For Author)\n\n- Test: Unit Test (S3UTILTest.is_s3_express_context; ASAN UT rebuild started)\n- Behavior changed: No, except avoiding false-positive S3 Express detection\n- Does this need documentation: No --- be/src/io/fs/s3_obj_storage_client.cpp | 18 ++++------------- be/src/io/fs/s3_obj_storage_client.h | 2 +- be/src/util/s3_util.cpp | 28 ++++++++++++++++++-------- be/src/util/s3_util.h | 1 + be/test/util/s3_util_test.cpp | 8 ++++++++ 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 3950f23bb034a9..1d1f146cd6244f 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -61,7 +61,6 @@ #include #include #include - #include #include @@ -121,29 +120,20 @@ using namespace Aws::S3::Model; static constexpr int S3_REQUEST_THRESHOLD_MS = 5000; namespace { -// S3 Express One Zone endpoints follow the pattern -// "*.s3express-..amazonaws.com" — substring match is sufficient -// for both the gateway and the s3express subdomain forms. -bool is_s3_express_endpoint(const std::string& endpoint) { - return endpoint.find("s3express") != std::string::npos; -} - // AWS expects the CRC32C value as the big-endian 4-byte representation, base64-encoded. Aws::String compute_crc32c_b64(std::string_view data) { uint32_t crc = crc32c::Crc32c(reinterpret_cast(data.data()), data.size()); uint8_t bytes[4] = {static_cast((crc >> 24) & 0xff), static_cast((crc >> 16) & 0xff), - static_cast((crc >> 8) & 0xff), - static_cast(crc & 0xff)}; + static_cast((crc >> 8) & 0xff), static_cast(crc & 0xff)}; return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::ByteBuffer(bytes, sizeof(bytes))); } } // namespace S3ObjStorageClient::S3ObjStorageClient(std::shared_ptr client, - const std::string& endpoint) + bool is_s3_express) : _client(std::move(client)), - _disable_content_md5(config::s3_disable_content_md5 || - is_s3_express_endpoint(endpoint)) {} + _disable_content_md5(config::s3_disable_content_md5 || is_s3_express) {} ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { @@ -545,4 +535,4 @@ std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOp expiration_secs); } -} // namespace doris::io \ No newline at end of file +} // namespace doris::io diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index ac048a65a98238..0216318936e4f4 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -33,7 +33,7 @@ class ObjClientHolder; class S3ObjStorageClient final : public ObjStorageClient { public: explicit S3ObjStorageClient(std::shared_ptr client, - const std::string& endpoint = {}); + bool is_s3_express = false); ~S3ObjStorageClient() override = default; ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override; diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 3fd802f5b8fe9d..028613f5533dc4 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -532,20 +532,17 @@ std::shared_ptr S3ClientFactory::_create_s3_client( // S3 Express buckets are identified by the --x-s3 suffix or an s3express endpoint. // Skip endpointOverride for these so the SDK resolves the bucket-specific endpoint // and manages CreateSession automatically. For all other buckets, keep the override. - const bool is_s3_express = s3_conf.endpoint.find("s3express") != std::string::npos || - s3_conf.bucket.find("--x-s3") != std::string::npos; - if (s3_conf.need_override_endpoint && !is_s3_express) { + const bool express = is_s3_express(s3_conf.bucket, s3_conf.endpoint); + if (s3_conf.need_override_endpoint && !express) { aws_config.endpointOverride = s3_conf.endpoint; } - aws_config.disableS3ExpressAuth = !is_s3_express; + aws_config.disableS3ExpressAuth = !express; auto new_client = std::make_shared( get_aws_credentials_provider(s3_conf), - Aws::MakeShared("S3Client"), - aws_config); + Aws::MakeShared("S3Client"), aws_config); - auto obj_client = - std::make_shared(std::move(new_client), s3_conf.endpoint); + auto obj_client = std::make_shared(std::move(new_client), express); LOG_INFO("create one s3 client with {}", s3_conf.to_string()); return obj_client; } @@ -807,4 +804,19 @@ std::string hide_access_key(const std::string& ak) { return key; } +bool is_s3_express(std::string_view bucket, std::string_view endpoint) { + constexpr std::string_view bucket_suffix = "--x-s3"; + if (bucket.ends_with(bucket_suffix)) { + return true; + } + + const auto scheme_pos = endpoint.find("://"); + if (scheme_pos != std::string_view::npos) { + endpoint.remove_prefix(scheme_pos + 3); + } + endpoint = endpoint.substr(0, endpoint.find('/')); + return endpoint.starts_with("s3express-") || + endpoint.find(".s3express-") != std::string_view::npos; +} + } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 5546db857c91b1..950600fe8a534f 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -63,6 +63,7 @@ extern bvar::LatencyRecorder s3_copy_object_latency; }; // namespace s3_bvar std::string hide_access_key(const std::string& ak); +bool is_s3_express(std::string_view bucket, std::string_view endpoint); int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); // Rebuild the S3 GET/PUT rate limiters if the related configs have changed. // Safe to call periodically; it is a no-op when nothing changed. diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index a87d0e9a2d2980..cba03bec576773 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -77,6 +77,14 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { EXPECT_EQ("xxxxxxxFODNN7xxxxxxx", result); } +TEST_F(S3UTILTest, is_s3_express_context) { + EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "")); + EXPECT_TRUE(is_s3_express("", "bucket.s3express-use1-az4.us-east-1.amazonaws.com")); + EXPECT_FALSE(is_s3_express("bucket--x-s3-suffix", "")); + EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path")); + EXPECT_FALSE(is_s3_express("bucket", "s3.us-east-1.amazonaws.com")); +} + // Verifies that check_s3_rate_limiter_config_changed() rebuilds the global GET rate // limiter when the related configs change. This is the behavior the cloud vault refresh // thread relies on to apply dynamically modified s3_get_* rate limiter configs without From 2e6d3acbeedb099459b1d1d1cdf937a75a41c84d Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 16 Jul 2026 20:15:56 +0800 Subject: [PATCH 04/13] [doc](s3) Add S3 Express One Zone implementation design ### What problem does this PR solve? Issue Number: N/A Related PR: #63409 Problem Summary: Document a production-ready design for S3 Express One Zone on Doris. The design audits the current PR, defines service-aware bucket capabilities, preserves ordinary S3 endpoint behavior, completes checksum and multipart flows across BE and FE, specifies unordered listing compatibility, rejects unsupported Cloud and Hadoop surfaces before I/O or persistence, and provides test, rollout, and data rollback gates. ### Release note None ### Check List (For Author) - Test: No need to test (documentation-only change; validated with git diff --cached --check and source-path consistency checks) - Behavior changed: No - Does this need documentation: No (this commit is the implementation design) --- goal.md | 1225 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1225 insertions(+) create mode 100644 goal.md diff --git a/goal.md b/goal.md new file mode 100644 index 00000000000000..607715f7c3612b --- /dev/null +++ b/goal.md @@ -0,0 +1,1225 @@ +# Apache Doris S3 Express One Zone 落地设计 + +## 1. 文档信息 + +| 项目 | 内容 | +| --- | --- | +| 状态 | Proposed | +| 最后核对日期 | 2026-07-16 | +| 实现基线 | take-over-63409，HEAD 0a193178c8 | +| 关联 PR | [apache/doris#63409](https://github.com/apache/doris/pull/63409) | +| 目标读者 | Doris BE、FE、Cloud、测试与发布维护者 | + +本文给出 Doris 支持 AWS S3 Express One Zone 的可实施方案。严格来说，S3 Express One Zone 是 storage class，Directory Bucket 是承载它的 bucket 类型；本文 P0 专指 Availability Zone 中使用 S3 Express One Zone 的 Directory Bucket。这里的“支持”不是简单地让一次 PutObject 成功，而是让 endpoint 解析、CreateSession、凭证刷新、checksum、multipart、ListObjectsV2、失败清理、普通 S3 兼容性和可观测性形成完整闭环。 + +本文中的 AWS 行为以 2026-07-16 的官方文档为准。S3 Directory Bucket 的能力仍在演进，开发和发布前必须重新核对文末的 AWS API 白名单与差异文档。 + +## 2. 结论先行 + +最终方案采用以下决策： + +1. 把 Directory Bucket 作为一种 S3 客户端能力模型，而不是通过“是否关闭 MD5”或 endpoint 子串做临时分支。 +2. Doris 只配置长期 IAM credentials 或既有 credentials provider。CreateSession、五分钟 session credentials 的缓存与刷新全部交给支持 S3 Express 的 AWS SDK，Doris 不自行实现 token manager。 +3. Directory Bucket 数据面不设置 endpointOverride，由 SDK 根据完整 bucket 名和 region 解析 Zonal endpoint；强制 HTTPS 和 virtual-hosted addressing，拒绝 path-style。 +4. Directory Bucket 写入统一显式使用 CRC32C。PutObject 发送对象 checksum；multipart 完整贯通 CreateMultipartUpload、UploadPart 和 CompleteMultipartUpload 的 part checksum。 +5. Directory Bucket 的 ListObjectsV2 不能沿用普通 S3 的 StartAfter 和字典序假设。Doris 使用 continuation token 扫描、客户端过滤和排序来保持现有 FileSystem 接口语义。 +6. 首个正式支持范围限定为 Doris 原生 S3 文件系统路径，包括 BE 读写以及与之配套的 FE 校验和列举。Cloud Storage Vault、Hadoop/S3A、预签名 URL、Access Point 不在首期支持声明中。 +7. 当前 PR 不能直接作为“Doris 已完整支持 S3 Express”合入。它已经验证了 BE 基础方向，但还缺少 FE、Cloud 边界、multipart 完整性、列举语义、失败 Abort、回归保护和真实 Directory Bucket 自动化测试。 + +## 3. 产品范围 + +### 3.1 首期 P0 支持矩阵 + +| Doris 使用面 | P0 状态 | 说明 | +| --- | --- | --- | +| BE 原生 S3 FileSystem exact-key 读、Head | 支持 | GetObject、HeadObject，由 C++ SDK 自动选择 Zonal endpoint 和 session auth | +| BE 小对象写入 | 支持 | PutObject，显式 CRC32C，不发送 Content-MD5 | +| BE 大对象写入 | 支持 | Create、UploadPart、Complete 全链路 CRC32C；失败时主动 Abort | +| BE 删除 | 支持 | DeleteObject；DeleteObjects 必须使用 flexible checksum | +| 原生 S3 前缀列举和 FE glob | 支持，但有代价 | 使用 continuation token 扫描相关目录；FE 客户端过滤、排序并应用 startAfter、maxFiles、maxBytes | +| FE S3 Resource 连通性检查 | 支持 | Java SDK v2，不强制通用 SigV4 signer，不 override Zonal endpoint | +| FE 新 S3 FileSystem 客户端 | 支持 | Java SDK v2，复用同一 credentials provider 和 bucket 分类规则 | +| 普通 AWS S3 | 保持现状 | endpoint、MD5、path-style 等既有行为不能回归 | +| MinIO、COS、OSS 等 S3-compatible | 保持现状 | 不因 bucket 名恰好以 --x-s3 结尾而切换到 AWS Express 模式 | + +### 3.2 P0 明确不支持 + +| 使用面 | 处理方式 | +| --- | --- | +| Cloud Storage Vault | FE 与 Meta Service 的创建、修改入口都 fail fast；原因见第 12 节 | +| 依赖 Hadoop S3A 的 External Catalog | 在确认所用 Hadoop/AWS SDK 版本支持 Directory Bucket 之前 fail fast | +| AWS Java SDK v1 路径 | 不支持；必须迁移到 Java SDK v2 后才能纳入 | +| Presigned URL | Directory Bucket 模式明确返回 NotSupported，完成独立鉴权和过期时间测试后再开放 | +| Directory Bucket Access Point | 不支持；其 host 和资源标识不同于直接 bucket | +| CopyObject、UploadPartCopy、RenameObject | 不作为 P0 对外承诺，实际调用面需要分别验证 | +| SSE-C、DSSE-KMS、对象 ACL、Object Lock、Requester Pays、对象 tag | Directory Bucket 不支持或不在 P0 | +| Doris 创建、删除 Directory Bucket | 不支持；bucket 由管理员预先创建 | +| Local Zone Directory Bucket | 不在 P0；P0 只声明 Availability Zone 中的 S3 Express One Zone | + +### 3.3 非目标 + +- 不通过 Doris 自研 HTTP 签名或 token 缓存替代 AWS SDK。 +- 不把 S3 Express 当成普通 S3 的透明性能开关。 +- 不承诺跨 AZ 的低延迟，也不在 Doris 内自动查询 IMDS 判断节点所在 AZ。 +- 不把单 AZ Directory Bucket 描述为普通 S3 的同等耐久性替代品。 +- 不在本次设计中改变 Doris 对象路径、文件格式或持久化元数据格式。 + +## 4. AWS 硬约束 + +### 4.1 Bucket、endpoint 与寻址 + +Availability Zone Directory Bucket 的完整名称为： + + ----x-s3 + +例如： + + analytics-hot--use1-az4--x-s3 + +zone-id 是全局一致的 Availability Zone ID，例如 use1-az4，不是账户内可能映射不同的 Availability Zone 名称。 + +数据面和控制面的 endpoint 不能混用： + +| 平面 | 典型操作 | endpoint | 寻址方式 | 认证 | +| --- | --- | --- | --- | --- | +| Zonal 数据面 | CreateSession、GetObject、PutObject、ListObjectsV2、multipart、Delete | s3express-..amazonaws.com | 只支持 virtual-hosted | 默认 session auth，由 SDK处理 | +| Regional 控制面 | 创建/删除 bucket、lifecycle、bucket policy 等 | s3express-control..amazonaws.com | path-style | 长期 IAM credentials 与 SigV4 | + +Doris P0 只需要对象数据面。客户端应把 bucket 和 region 交给 SDK endpoint rules，不能把用户填写的 endpoint 直接覆盖成 Zonal host，更不能把 s3express-control 当成对象 endpoint。 + +### 4.2 CreateSession + +- IAM principal 必须拥有目标 Directory Bucket 上的 s3express:CreateSession。 +- SDK 根据 --x-s3 bucket 名自动请求 bucket-scoped session credentials。 +- session credentials 有效期为五分钟；SDK 负责缓存、并发复用和临近过期刷新。 +- Doris 只保存或传递长期 credentials provider 所需的信息，不把 SDK 生成的 session access key、secret key 或 token 写入 FE、Meta Service、配置、日志或 RPC。 +- SDK 对少数使用长期 IAM 签名的操作有自己的签名选择逻辑，Doris 不应强制指定 AwsS3V4Signer 或自行指定 service name。 + +最小读写 IAM 示例： + + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3express:CreateSession", + "Resource": "arn:aws:s3express:us-east-1:123456789012:bucket/analytics-hot--use1-az4--x-s3" + } + ] + } + +生产环境应使用角色和最小权限策略。若需要只读 session，应通过 AWS 支持的 s3express:SessionMode 条件约束 IAM，而不是让 Doris 修改 session token。 + +### 4.3 Checksum 与 ETag + +- Directory Bucket 的 PutObject 和 UploadPart 不支持 Content-MD5。 +- CRC32、CRC32C、SHA-1、SHA-256 可用于数据完整性校验；AWS 推荐高性能场景使用 CRC32 或 CRC32C。 +- 本设计固定选择 CRC32C，避免不同 SDK 版本的隐式默认值变化，并复用 Doris 已有的硬件优化 CRC32C 实现。 +- Directory Bucket 的 ETag 是不透明标识，不能假设为对象 MD5。 +- DeleteObjects 的 XML request body 必须带 Content-MD5 或 flexible checksum；Directory Bucket 路径使用 CRC32C。 + +### 4.4 Multipart + +- Directory Bucket 的 part number 必须从 1 开始连续递增；Complete 时不允许空洞。 +- part 可以并发上传，Doris 在 Complete 前按 part number 排序并验证连续性。 +- 一旦声明 multipart checksum algorithm，各 UploadPart 和 CompleteMultipartUpload 必须携带一致的 part checksum。 +- 未完成的 multipart 会持续占用存储并计费。应用必须主动 Complete 或 Abort，lifecycle 只能作为兜底。 + +### 4.5 ListObjectsV2 + +Directory Bucket 与普通 S3 的关键差异： + +- 返回结果不保证按 key 字典序排序。 +- 不支持 StartAfter。 +- Prefix 只支持以 / 结尾的值。 +- Delimiter 只支持 /。 +- 分页必须使用服务端返回的 opaque continuation token。 +- 列表中可能出现仅由未完成 multipart 产生的 prefix。 + +因此，任何依赖“服务端排序 + last key 续扫”的 Doris 代码都必须改造，不能只修改 endpoint 和鉴权。 + +### 4.6 能力限制 + +- 不支持 S3 Versioning。 +- 当前 AWS API 支持 Directory Bucket lifecycle，但只支持对象过期和取消未完成 multipart；不支持非当前版本过期、transition 和 tag filter。 +- P0 使用 bucket 默认 SSE-S3。SSE-KMS 需要单独验证 CreateSession、bucket 默认加密和请求 header 一致性后再开放。 +- StorageClass header 应省略或使用 EXPRESS_ONEZONE，不能发送 STANDARD。 +- Access Point 已是 AWS 独立能力，但不在 P0；不能误写成 AWS 永久不支持。 + +## 5. 当前 PR 评估 + +### 5.1 当前分支 + +take-over-63409 相对 upstream/master 的 merge-base e05c331da1 有三个提交： + +1. 4096f12a4f：PutObject 和 UploadPart 增加 CRC32C，并为 Express 关闭 Content-MD5。 +2. 98b7adffee：AWS SDK C++ 1.11.219 升级到 1.11.400，并改用 S3ClientConfiguration。 +3. 0a193178c8：集中和收紧 S3 Express 检测。 + +原 GitHub PR #63409 仍只有前两个提交；第三个检测修复尚未进入原 PR。原 PR 当前为 Changes requested。 + +### 5.2 已经正确的方向 + +- be/src/util/s3_util.cpp 开始使用 S3ClientConfiguration 和 S3EndpointProvider。 +- Directory Bucket 模式不再设置 endpointOverride。 +- disableS3ExpressAuth=false 时可由 SDK 接管 CreateSession。 +- be/src/io/fs/s3_obj_storage_client.cpp 已验证显式 CRC32C 的基本写入方向。 +- 普通 S3 仍保留 Content-MD5 分支。 +- common/cpp/aws_logger.h 已适配新 SDK 的 vaLog 接口。 + +### 5.3 必须修正的事实描述 + +AWS SDK C++ 1.11.219 并非完全不支持 S3 Express。AWS SDK C++ 维护者说明 S3 Express customizations 从 1.11.212 已合入；1.11.219 源码中已经存在 CreateSession、S3 Express signer/provider、disableS3ExpressAuth 和 endpoint rules。旧的 S3Client 构造函数在该版本也会创建 endpoint provider 和 Express signer provider。 + +因此： + +- 可以继续采用 1.11.400 作为 Doris 的固定版本，因为 AWS 建议采用包含后续 bug fix 的较新版本。 +- PR 描述不能把“1.11.219 没有 S3 Express 能力”作为根因。 +- 改成显式 S3ClientConfiguration 的价值是让配置和 endpoint provider 更清楚、可测试，而不是它单独开启了 Express。 +- 发布前要用 pinned SDK 的集成测试证明 endpoint、session refresh 和 checksum，而不是依赖版本号推断。 + +### 5.4 当前实现缺口和回归风险 + +| 问题 | 具体触发链 | 设计要求 | +| --- | --- | --- | +| 覆盖面只有 BE 基础路径 | FE 和 Cloud 有独立 S3 client builder | 分别改造或明确排除，不能宣称 Doris 全面支持 | +| 检测仍把 bucket/endpoint 字符串作为布尔值 | 第三方 bucket 恰好以 --x-s3 结尾会被跳过自定义 endpoint | 引入 service-aware capability resolver，联合 bucket、官方 endpoint authority 和 override mode | +| disableS3ExpressAuth 与自研检测绑定 | AWS 普通 bucket 被强制关闭 SDK 默认能力，第三方行为与 endpoint 子串耦合 | 官方 AWS service 保持 SDK 默认 false；自定义 endpoint client 明确禁用 | +| 新配置从默认值重建 | be/src/util/s3_util.cpp 直接默认构造 S3ClientConfiguration | 从现有 ClientConfiguration 转换，保留 CA、timeout、retry 等语义 | +| payload signing 策略变化 | 旧构造显式 Never，新构造使用默认 RequestDependent | 普通路径显式保持现状，并做回归与性能测试 | +| s3_disable_content_md5 是动态全局配置 | 配置只在 cached S3ObjStorageClient 构造时快照 | P0 删除该用户开关；checksum 由 bucket capability 决定 | +| cache key 不完整 | S3ClientConf::get_hash() 缺 need_override_endpoint 和新策略 | 所有影响 client 行为的字段进入 key | +| multipart checksum 只有 UploadPart | Create 未声明算法、Complete 只传 ETag | 完整传播 CRC32C | +| 写失败不 Abort | S3FileWriter::_abort 仅声明，底层无接口 | 增加 abort API 并覆盖取消、失败和并发上传收敛 | +| DeleteObjects 未覆盖 | Directory Bucket request body 不接受既有 MD5 假设 | 显式 CRC32C 或验证 SDK 注入，必须有 request capture test | +| List 仍依赖普通 S3 语义 | FE glob 直接下推 StartAfter，Prefix 可不以 / 结尾 | continuation token + 本地过滤、排序 | +| 可能泄露 token | S3ClientConf::to_string() 直接输出 token | token 和 externalId 必须遮蔽或完全移除 | +| 默认 HTTP 风险 | s3_client_http_scheme 和 S3Resource 历史路径可生成 HTTP | Directory Bucket 强制 HTTPS | +| 测试仅覆盖检测 | 当前新增测试只有少量 is_s3_express case | 增加 client、request、multipart、list、refresh 和真实 AWS 测试 | + +### 5.5 Doris code-review 关键检查点结论 + +| 检查点 | 当前 PR 结论 | +| --- | --- | +| 目标是否完成、是否有测试证明 | 仅完成 BE PutObject/UploadPart 的基础尝试；没有证明完整 Doris 调用链，也没有 session refresh、multipart、list 或真实 AWS 自动化测试 | +| 修改是否最小清晰 | 差异本身较小，但用全局 MD5 开关和 bool Express 状态承载多种能力，抽象不足 | +| 并发 | S3FileWriter 并发上传 part；当前没有等待所有 part 后 Abort 的失败收敛设计。SDK session provider 的并发复用也未测试 | +| 生命周期 | 动态配置在 cached client 构造时快照；client、五分钟 session、multipart upload 的创建与销毁生命周期尚未闭环 | +| 新增配置 | s3_disable_content_md5 声明为 mutable，但运行时更新对已有 client 不生效，不能按当前形态保留 | +| 兼容与滚动升级 | 没有 wire/storage format 变化，但重建 S3ClientConfiguration 可能改变 timeout 和 payload signing；旧 BE 不能用于 Directory Bucket | +| 平行调用路径 | FE 两套 Java builder、Hive BE-upload/FE-complete、Cloud recycler/client、特殊 BE client 创建点没有同步覆盖 | +| 特殊条件判断 | bucket/endpoint 字符串判断不能区分官方 AWS 与自定义 S3-compatible endpoint | +| 测试覆盖 | 新增测试只覆盖少量识别输入；请求字段、错误路径、负例和端到端覆盖不足 | +| 可观测性 | 没有 Directory Bucket 专用 metrics；现有 S3ClientConf::to_string() 还会输出明文 token | +| 持久化与协议 | bucket capability 无需持久化；但 BE 上传、FE 完成的 Hive multipart 必须增加可选 Thrift checksum 字段 | +| 数据写入正确性 | 小对象 CRC32C 方向正确，但 multipart checksum、DeleteObjects checksum 和失败 Abort 不完整 | +| 性能 | client/session 复用方向正确；需验证 payload signing、CRC32C CPU 和 Directory Bucket 全量 list 的成本 | + +## 6. 统一能力模型 + +### 6.1 C++ 数据结构 + +在 BE 和 Cloud 都可依赖的 common/cpp 层定义纯数据类型和无网络副作用的 resolver。名称可按现有构建组织调整，建议语义如下： + + enum class S3BucketType { + GENERAL_PURPOSE, + DIRECTORY + }; + + enum class S3EndpointMode { + EXPLICIT_OVERRIDE, + AWS_SDK_RULES + }; + + enum class S3ChecksumPolicy { + CONTENT_MD5, + CRC32C + }; + + struct S3BucketCapabilities { + S3BucketType bucket_type; + S3EndpointMode endpoint_mode; + S3ChecksumPolicy checksum_policy; + bool require_https; + bool require_virtual_addressing; + bool supports_start_after; + bool list_is_lexicographic; + bool supports_versioning; + bool supports_presign; + }; + +resolver 输入必须包含： + +- storage provider（若当前 surface 有该信息） +- 完整 bucket 名 +- endpoint +- region +- use_path_style 或 use_virtual_addressing +- need_override_endpoint 或等价的“官方 AWS 服务/自定义 S3-compatible 服务”信息 +- 当前调用面是否声明支持 Directory Bucket + +resolver 输出不可只返回 bool。调用者应基于 capability 做 endpoint、auth、checksum、list 和功能门禁，避免散落多个 is_s3_express 判断。 + +### 6.2 判定真值表 + +| 服务意图 | Bucket | Endpoint/override | 结果 | +| --- | --- | --- | --- | +| AWS 服务 | 完整 --x-s3 后缀 | 空、标准 AWS regional endpoint 或匹配的官方 Zonal endpoint | DIRECTORY | +| AWS 服务 | 非 --x-s3 | 官方 s3express Zonal/control endpoint | 配置错误，fail fast | +| 自定义 S3-compatible 服务 | 任意名称，包括 --x-s3 后缀 | 自定义 endpoint，明确需要 override | GENERAL_PURPOSE，不自动切 Express | +| 非 S3 provider | 任意名称 | 官方 s3express endpoint | 配置错误，fail fast | + +这里的 AWS 服务不能只由用户属性 provider 或 C++ 的 ObjStorageType::AWS 判断。FE 合法的通用 S3 provider 值是 S3，进入 C++ 后又可能映射成 ObjStorageType::AWS；MinIO 等 S3-compatible 配置会走同一类枚举。resolver 必须结合 need_override_endpoint 和 endpoint authority：明确使用自定义 endpoint override 的客户端按 GENERAL_PURPOSE 处理；只有指向官方 AWS 服务且 bucket 后缀合法时才进入 DIRECTORY。endpoint 仍然不能单独作为 Directory Bucket 的判定依据，不能因为字符串中出现 s3express 就启用 session auth。 + +官方 endpoint 的识别应复用 AWS partition/endpoint metadata 或集中维护的 parser，不能把 .amazonaws.com 写死后遗漏其他 partition，也不能维护会快速过期的 region/AZ allowlist。 + +### 6.3 Bucket 解析 + +- 只接受完整后缀 --x-s3。 +- 从最后两个双连字符段解析 zone-id，不维护静态 AZ allowlist。 +- region 必填。 +- 若 endpoint 同时包含 region 或 zone-id，发现明显不一致时返回包含 bucket、region、endpoint 的可操作错误。 +- 不仅检查 endpoint 字符串；最终 endpoint 仍由 SDK rules 生成。 + +### 6.4 跨语言一致性 + +FE Java 无法直接复用 C++ 类型，因此应： + +1. 在 S3URI/S3Properties/S3FileSystemProperties 附近实现同样的 service-aware resolver。 +2. 增加一份跨语言测试向量，覆盖 bucket、provider（若有）、endpoint、override mode、region、path-style 和预期结果。 +3. C++ 和 Java 单测读取同一组向量，防止后续规则漂移。 + +P0 不为 bucket type 新增 protobuf、Thrift 或持久化字段。各 surface 使用已有 bucket、region、endpoint、path-style，并结合本地 endpoint override 规则确定性派生 capability；不能假设 provider 已在所有路径传递，因为 S3FileSystemProperties 和 S3Properties.getS3TStorageParam() 当前不会完整传递它。 + +multipart checksum 是另一件事：BE 上传、FE 完成的 Hive 写入必须新增可选 Thrift 字段，详见第 8.5 节。可选字段保证旧消息可解析，但不代表混合版本可以启用 Directory Bucket。 + +### 6.5 FE 的两级解析 + +S3FileSystemProperties.bucket 在部分调用面可以为空，实际 bucket 来自每次 remotePath；S3ObjStorage 当前也只维护一个 lazy S3Client。因此 FE 不能在 buildClient 时把整个 client 固化成某个 bucket type。 + +FE 使用两级模型： + +1. service-level config 在创建 S3ObjStorage 时确定： + - AWS_SDK_ROUTED：endpoint 为空或为可安全归一化的标准 regional endpoint；允许 Directory Bucket。 + - AWS_EXPLICIT_ENDPOINT：FIPS、dualstack、accelerate 或其他必须保留的显式 AWS endpoint；P0 只允许普通 bucket。 + - CUSTOM_S3_COMPATIBLE：保留 endpointOverride，禁用 Express session auth。 +2. request-level capability 在每个方法解析 remotePath 后，根据实际 bucket、service-level config、region 和 path-style 派生： + - AWS_SDK_ROUTED + --x-s3 为 DIRECTORY。 + - AWS_SDK_ROUTED + 普通 bucket 为 GENERAL_PURPOSE。 + - AWS_EXPLICIT_ENDPOINT + 普通 bucket 为 GENERAL_PURPOSE；遇到 --x-s3 时 fail fast。 + - CUSTOM_S3_COMPATIBLE 下所有 bucket 都按 GENERAL_PURPOSE。 + +为保证普通 AWS S3 不回归，S3ObjStorage 将当前单一 lazy client 改成 clientFor(bucket)： + +- general client 完整保留已有 endpointOverride、FIPS/dualstack、path-style 和兼容配置。 +- directory client 只在 AWS_SDK_ROUTED 的 Directory Bucket 请求出现时惰性创建，固定为无 endpointOverride、HTTPS、virtual-hosted、Express auth enabled。 +- 两个 client 复用同一长期 credentials provider，但生命周期和配置缓存分开。 +- 一个 directory client 可以服务同 region 的多个 Directory Bucket，SDK 的 Express identity cache 按 bucket 和 credentials identity 隔离 session。 + +Put、multipart、List、Copy、presign 等方法必须使用当前 URI 的 request-level capability 和 clientFor(bucket)，不能把第一次请求的 bucket type 缓存在 S3ObjStorage 上，也不能为每个对象创建新 client。 + +## 7. 客户端与请求数据流 + + FE properties / URI / storage resource + | + v + service-aware capability resolver + | + +---- unsupported surface: fail fast + | + v + existing PB / Thrift storage parameters + only long-term credentials configuration + | + v + BE S3ClientFactory client cache + | + v + AWS SDK endpoint rules + credentials provider + | + +---- CreateSession + | | + | +---- SDK-only five-minute session credentials + | + v + bucket-specific Zonal data endpoint + | + +---- Get / Head / List + +---- Put / Multipart / Delete + +Hive multipart 的事务提交链路另有一条 BE 到 FE 的 TS3MPUPendingUpload 消息，只传算法、ETag 和 part checksum 等完成上传所需元数据，仍然不传 session credentials。 + +关键生命周期： + +- client 以 bucket、region、service kind、endpoint mode、credentials provider identity 等为 key 长期缓存。 +- 首次 Directory Bucket 请求触发 SDK 获取 session。 +- 并发请求复用同一个 client 和 session provider。 +- SDK 在 session 过期前刷新；Doris 不观察和持久化 token。 +- client 销毁时 session credentials 只随进程内对象释放。 + +## 8. BE 详细改造 + +### 8.1 S3ClientConf 与 cache key + +文件： + +- be/src/util/s3_util.h +- be/src/util/s3_util.cpp + +改造： + +1. 在 S3ClientConf 初始化完成且 bucket 已知后调用 capability resolver。 +2. get_hash() 至少加入 need_override_endpoint、bucket type、endpoint mode、checksum policy、scheme、addressing style。 +3. 不继续用简单 XOR 拼接易交换字段的 hash；按字段顺序使用 hash combine。 +4. credentials provider 类型、role ARN、external ID、session token identity 必须继续参与隔离。 +5. to_string() 只能输出遮蔽后的 access key；token、secret key 和 external ID 不输出明文。 +6. client cache 不跨不同 bucket 复用 Directory Bucket client，确保 session credentials 的 bucket scope 不被混用。 + +### 8.2 S3ClientFactory + +文件： + +- be/src/util/s3_util.cpp + +普通 S3 路径： + +- 保留当前 endpointOverride 规则。 +- 保留 useVirtualAddressing、timeout、CA、retry 和 payload signing 的原有语义。 +- S3-compatible provider 不因名称命中 --x-s3 而改变行为。 + +Directory Bucket 路径： + +- 从 S3ClientFactory::getClientConfiguration() 返回的现有公共 ClientConfiguration 通过 converting constructor 构造 S3ClientConfiguration，避免丢失 aws_client_request_timeout_ms 等公共默认值。 +- region 使用用户配置。 +- scheme 强制 HTTPS；若配置显式要求 HTTP，直接返回配置错误。 +- useVirtualAddressing 强制 true；若用户配置 path-style，直接返回配置错误。 +- endpointOverride 保持空。 +- disableS3ExpressAuth 保持 false。 +- 使用 S3EndpointProvider。 +- credentials provider 仍是 Doris 已有 static、default chain、web identity、instance profile 或 assume-role provider。 +- 对普通路径显式保留 PayloadSigningPolicy::Never；Directory Bucket 路径让 SDK 的 Express customization 决定必要的签名行为，并用性能测试验证没有重复 payload hash。 + +不要把 disableS3ExpressAuth 设置成 !is_directory。建议规则是： + +- 官方 AWS 服务：false，允许 SDK 根据实际 bucket 选择。 +- 自定义 S3-compatible endpoint：true，防止 AWS SDK 根据特殊 bucket 名误启用 Express。 + +### 8.3 特殊构造路径审计 + +所有在 bucket 未知时创建 S3ClientConf 的路径必须修正。例如 be/src/exprs/function/ai/embed.h 应先解析 URI 并写入 bucket，再创建 client。审计要求： + + rg "S3ClientConf|create_s3_client|S3ClientFactory" be cloud + +每个直接构造 client 的路径都必须归入以下之一： + +- 使用统一 factory 和 capability resolver。 +- 明确仅支持普通 S3，并在 Directory Bucket 输入时 fail fast。 +- 有独立的 Directory Bucket 实现和测试。 + +### 8.4 PutObject + +文件： + +- be/src/io/fs/s3_obj_storage_client.h +- be/src/io/fs/s3_obj_storage_client.cpp + +改造： + +- S3ObjStorageClient 持有 S3BucketCapabilities，而不是 is_s3_express 和 _disable_content_md5 两个布尔值。 +- DIRECTORY 时： + - SetChecksumAlgorithm(CRC32C) + - 计算 Base64 编码的 CRC32C 并 SetChecksumCRC32C + - 不 SetContentMD5 +- GENERAL_PURPOSE 时保持当前 Content-MD5 行为。 +- 对空对象、非对齐 buffer、最大单次 PutObject 边界增加已知向量测试。 +- 不把 ETag 与本地 MD5 比较。 + +当前 s3_disable_content_md5 全局动态配置不进入最终 P0。若未来确实需要为第三方存储配置 checksum，应设计为每个 resource 的 AUTO、MD5、CRC32C、NONE 枚举，并贯穿 FE、PB/Thrift、BE 及 cache key，不能复用当前全局布尔值。 + +### 8.5 Multipart 完整校验 + +相关文件： + +- be/src/io/fs/obj_storage_client.h +- be/src/io/fs/s3_obj_storage_client.cpp +- be/src/io/fs/s3_file_writer.cpp +- be/src/io/fs/s3_file_writer.h + +Directory Bucket 流程： + +1. CreateMultipartUpload 设置 ChecksumAlgorithm=CRC32C。 +2. UploadPart 为每个 part 计算并发送 CRC32C。 +3. 校验服务端返回的 ChecksumCRC32C 与本地值一致；缺失或不一致均返回失败。 +4. ObjectStorageUploadResponse 增加可选 checksum_crc32c。 +5. ObjectCompleteMultiPart 增加可选 checksum_crc32c。 +6. S3FileWriter 保存 part number、ETag、CRC32C。 +7. CompleteMultipartUpload 的每个 CompletedPart 同时设置 ETag、PartNumber、ChecksumCRC32C。 +8. Complete 前排序并断言 part number 为 1..N 连续序列；违反不变量时失败，不能跳过空洞继续提交。 + +普通 S3FileWriter 在同一 BE 内完成上传时，上述内部结构足够；但 Hive 写入还存在“BE 上传 part，FE 提交事务后 Complete”的跨进程链路： + +- gensrc/thrift/DataSinks.thrift 的 TS3MPUPendingUpload 当前只有 bucket、key、upload_id 和 etags。 +- be/src/exec/sink/writer/vhive_partition_writer.cpp 只把 part ETag 写入 Thrift。 +- fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java 在 FE 发起 Complete。 +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java 当前把 ETag map 转成 UploadPartResult。 + +P0 必须给 TS3MPUPendingUpload 增加可选字段： + + 5: optional TObjectStorageChecksumAlgorithm checksum_algorithm + 6: optional map part_checksums + +其中 P0 枚举至少包含 CRC32C，part_checksums 的 key 与 etags 的 part number 一致。对应改造： + +1. VHivePartitionWriter 从 S3FileWriter.completed_parts() 同时填入算法和每个 part 的 CRC32C。 +2. HMSTransaction 把两个新字段交给 ObjFileSystem。 +3. ObjFileSystem 构造带 checksum 的 UploadPartResult。 +4. S3ObjStorage 在 Complete 时为每个 CompletedPart 设置 CRC32C。 +5. Directory Bucket 模式若新字段缺失，FE 必须拒绝 Complete 并走 Abort；不能降级成只传 ETag。 +6. 普通 S3 和旧消息可以不设置可选字段。 + +这些改动不改变持久化对象格式，但改变了可选 Thrift 消息。新代码可解析旧消息；语义上仍不允许混合版本启用 Directory Bucket，因为旧 FE 会忽略新字段，旧 BE 也不会生成它们。Azure、OSS、COS、OBS 等实现不使用 checksum 字段，但必须继续兼容现有构造。 + +若 pinned C++ SDK 的 request model 对 composite checksum 有额外约束，以 SDK 1.11.400 的实际模型和真实 AWS 测试为准；不能退化成“只要 UploadPart 成功就视为完整支持”。 + +### 8.6 AbortMultipartUpload + +当前 S3FileWriter::_abort 只有声明。P0 必须： + +1. 给 ObjStorageClient 增加 abort_multipart_upload。 +2. S3 实现调用 AWS AbortMultipartUpload；NoSuchUpload 视为幂等成功。 +3. 等待所有 in-flight UploadPart future 收敛后再 Abort，避免 part 上传与 Abort 竞态。 +4. Create 成功后的任一上传、Complete、取消或 close 失败都进入 Abort。 +5. 若业务操作失败且 Abort 也失败，返回保留原始错误并附带 cleanup 错误的 Status。 +6. 析构函数不执行可能阻塞或抛异常的网络请求，只记录未正常 close 的 metric 和 warning。 +7. 服务端 AbortIncompleteMultipartUpload lifecycle 作为最终兜底，不替代应用主动清理。 + +### 8.7 DeleteObjects + +审计 BE 和 Cloud 的批量删除实现： + +- Directory Bucket 使用 CRC32C 校验 XML request body。 +- 不设置 Content-MD5。 +- request capture 单测必须断言 algorithm 和 checksum header。 +- 保留每个 object 的失败明细；部分失败不能静默当作成功。 + +### 8.8 读取 + +- GetObject、HeadObject 由 SDK 处理 endpoint 和 session auth。 +- P0 不改变 Doris range read 语义。 +- 若启用 ChecksumMode，必须验证 ranged GET 的 AWS 行为；不能要求服务端对任意 range 返回完整对象 checksum。 +- TLS 是读取传输保护的最低要求。 +- 所有使用 ETag 做缓存 identity 的代码只把它当不透明字符串。 + +## 9. ListObjectsV2 兼容方案 + +### 9.1 现有冲突 + +fe-filesystem 的 FileSystem 接口把 startAfter 定义为字典序游标；S3CompatibleFileSystem.globListWithLimit 当前把它直接下推给 ListObjectsV2，并按服务返回顺序计算 maxFiles、maxBytes 和 GlobListing.maxFile。这在 Directory Bucket 上不成立。 + +相关文件： + +- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/FileSystem.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectListOptions.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java + +仅把 prefix 扩展到父目录还不够，因为： + +- StartAfter 请求会被拒绝。 +- 返回顺序不稳定，提前应用 maxFiles 或 maxBytes 会漏掉字典序更小的 key，并产生错误的 maxFile 游标。 +- 原始最长非通配 prefix 可能不以 / 结尾。 + +### 9.2 P0 算法 + +输入： + +- originalPrefix：Doris 语义中的最长前缀 +- directoryPrefix：originalPrefix 截断到最后一个 /，没有 / 时为空 +- globPattern +- startAfter +- maxFiles +- maxBytes + +执行： + +1. 只向 AWS 发送以 / 结尾的 directoryPrefix。 +2. 不发送 StartAfter。 +3. 用 NextContinuationToken 读取到 IsTruncated=false。 +4. 对每一页在客户端过滤 originalPrefix 和 globPattern。 +5. 在客户端应用 key > startAfter。 +6. 收集匹配对象的 key、size 和 modification time，并按 Doris 既有 key comparator 排序。 +7. 按排序结果依次加入 FileEntry；加入当前对象后，如果 files.size 达到 maxFiles 或 totalSize 达到 maxBytes，则结束当前返回页。这与现有“包含触发阈值的对象”语义一致。 +8. GlobListing.maxFile 设置为返回页后的第一个匹配 key；若不存在下一项，则保持现有契约，使用最后一个已返回 key。 +9. maxFiles 有值且 maxBytes 未启用时，可以用大小为 maxFiles+1 的最大堆优化内存；只要 maxBytes 启用，阈值依赖排序后对象大小，最坏情况必须收集全部匹配项后排序，不能套用单一 limit 堆算法。 +10. 多个服务 prefix 的结果需要去重后再统一排序；Directory Bucket 路径通常只使用扩大的一个 directoryPrefix。 +11. 全程响应查询取消；超过现有内存限制时返回 ResourceLimit 错误，不能静默截断。 + +复杂度： + +- 服务请求数与 directoryPrefix 下的总对象数相关。 +- 只有 maxFiles 限制时可把内存优化为 O(maxFiles)，CPU 为 O(N log maxFiles)。 +- maxBytes 启用或没有限制时，最坏内存为 O(M)，排序 CPU 为 O(M log M)，M 是匹配对象数。 + +这是保持现有 FileSystem 契约的正确性成本。文档应建议用户为高频列举使用稳定、以 / 结尾且选择性高的目录前缀。 + +### 9.3 BE 与 Cloud 的不同契约 + +BE S3ObjStorageClient.list_objects 返回完整 FileInfo 集合，没有 FE GlobListing 的 maxFiles、maxBytes、maxFile 契约。BE 只需要： + +- continuation token 翻完所有服务页。 +- 任意非目录 prefix 先扩大到父目录，再客户端精确过滤。 +- 不承诺或依赖服务返回顺序；只有上层契约要求时才排序。 +- 未完成 multipart 产生的 CommonPrefixes 不当成已提交对象。 + +BE 和 Cloud 的递归删除也不能直接复用 FE 排序算法。尤其不能一边按 opaque continuation token 翻页，一边删除已经列出的对象后继续使用原 token。Directory Bucket 的安全算法是： + +1. 从空 continuation token 开始扫描，客户端过滤目标 prefix。 +2. 收集最多一个 DeleteObjects batch 的匹配 key；如果前几页没有匹配项，继续翻页，不能提前结束。 +3. 删除该 batch 后丢弃 continuation token，从头重新扫描。 +4. 重复直到一次完整扫描找不到匹配对象。 +5. 每个 DeleteObjects request 使用 CRC32C，并检查部分失败。 + +Cloud recycler 的 list_prefix、递归删除和 checker 只需要集合完整性和取消/重试语义，不需要 FE 的字典序分页结果。三条调用链共享 AWS 协议约束和 prefix 过滤 helper，但各自保留自己的返回契约。这里的 Cloud 算法只描述第 12.2 节的独立 P1：P0 不让 Directory Bucket 请求进入 Cloud recycler，必须在 Vault 持久化前拒绝。 + +## 10. FE 详细改造 + +### 10.1 S3URI 与 S3Properties + +相关文件： + +- fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +- fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java + +现有 S3URI 已有 directory bucket 辅助逻辑，可以保留 bucket suffix 解析，但必须升级为 service-aware capability resolver。校验时： + +- OFFICIAL_AWS service + --x-s3 才进入 Directory Bucket。 +- region 必填。 +- path-style=false。 +- endpoint 必须是 HTTPS。 +- 推荐 endpoint 配置为标准 AWS regional endpoint，例如 https://s3.us-east-1.amazonaws.com；它只作为 region/配置提示，不下传为 Directory Bucket 的 endpoint override。 +- 对 control endpoint、region/zone 冲突给出明确错误。 + +若某个 surface 在校验阶段拿不到 bucket，不能只看 endpoint 猜测，也不能要求 S3FileSystemProperties.bucket 必填。先区分 AWS_SDK_ROUTED、AWS_EXPLICIT_ENDPOINT、CUSTOM_S3_COMPATIBLE，再在每次 remotePath 解析出 bucket 后确定 request-level capability。 + +### 10.2 fe-filesystem-s3 + +相关文件： + +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjStorage.java + +当前代码注释说“仅为非 AWS endpoint 设置 endpointOverride”，但实现对任何非空 endpoint 都 override。不能为支持 Directory Bucket 而一律删除官方 AWS endpointOverride，因为普通 bucket 可能依赖 FIPS、dualstack 或显式 regional endpoint。改造为： + +- general client： + - 完整保留当前 endpointOverride、path-style、FIPS/dualstack 和兼容 signer 行为。 + - CUSTOM_S3_COMPATIBLE 调用 disableS3ExpressSessionAuth(true)。 +- directory client： + - 只允许 AWS_SDK_ROUTED。 + - 不调用 endpointOverride。 + - region 必填。 + - disableS3ExpressSessionAuth(false)。 + - 不设置自定义 signer。 + - 使用 buildCredentialsProvider() 返回的同一 provider。 + - 强制 HTTPS 和 virtual-hosted，让 Java SDK v2 解析 Zonal endpoint/auth。 +- AWS_EXPLICIT_ENDPOINT 遇到 Directory Bucket 时 fail fast；P0 不静默丢弃用户指定的 FIPS/dualstack 语义。 + +每次请求解析 bucket 后由 clientFor(bucket) 选择 general 或 directory client。若当前 remotePath bucket 是 DIRECTORY 且 usePathStyle=true，在发送请求前 fail fast；普通 bucket 保留既有 path-style 行为。 + +S3ObjStorage 自身也执行 S3Resource ping 的 Put 和 multipart，不能只修 client builder： + +- 每个公开方法解析 S3Uri 后调用 capabilitiesFor(uri.bucket())，不缓存第一次请求的 bucket type。 +- Directory PutObject 显式选择 CRC32C，让 Java SDK 计算或发送 checksum，并用 request interceptor 测试实际 header。 +- Directory CreateMultipartUpload 声明 CRC32C。 +- Directory UploadPart 返回并保存 ChecksumCRC32C。 +- Java SPI 的 UploadPartResult 增加可选 checksum 字段，并保留现有两参数构造以避免影响 OSS、COS、OBS、Azure。 +- CompleteMultipartUpload 为每个 CompletedPart 传递 CRC32C。 +- CopyObject、presign 等未纳入 P0 的操作在 Directory Bucket 模式 fail fast。 +- 普通 S3-compatible 请求保持现有行为。 + +general/directory 两个 lazy client 各自使用线程安全的初始化方式。测试要覆盖并发首次请求，确认每种配置只创建一个 client，且两种配置不会串用。 + +### 10.3 fe-core S3Util + +相关文件： + +- fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java + +该文件使用 AWS Java SDK v2，但当前 builder： + +- 无条件 endpointOverride。 +- 强制 AwsS3V4Signer。 +- 可开启 path-style。 + +把 builder 收敛为接收规范化的 service-level config，并由已知 bucket 选择配置： + +- Directory Bucket 使用 AWS_SDK_ROUTED：不 override endpoint、不强制 signer，并启用 Express session auth。 +- 普通 AWS 或自定义 S3-compatible 保留现有显式 endpoint、signer 和 chunkedEncoding 兼容设置；自定义 endpoint 禁用 Express session auth。 +- AWS_EXPLICIT_ENDPOINT + Directory Bucket 在构建前 fail fast。 +- HeadBucket 连通性检查必须使用完整 bucket，并在调用前用 request-level capability 拒绝 Directory Bucket 的 path-style/HTTP 配置。 + +不建议继续扩散多个 buildS3Client overload。先收敛到一个接收规范化配置对象的内部 builder，再让旧 overload 委托它。 + +### 10.4 S3Resource + +相关文件： + +- fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java + +改造： + +- Directory Bucket 不允许缺省成 http://。 +- ping test 的 ListObjects prefix 必须使用父目录 slash prefix，并在客户端过滤目标 key。 +- ping test 复用现有 ObjStorage.abortMultipartUpload API；任何失败路径都 Abort，并把 Abort 失败附加到原始错误，不能像当前实现一样吞掉 cleanup 异常。 +- 错误信息区分 InvalidDirectoryBucketConfiguration、CreateSessionAccessDenied、UnsupportedS3Surface。 + +### 10.5 Presigned URL + +S3ObjStorage.getPresignedUrl 当前使用 static basic credentials 创建 presigner，和实际 client 的 default chain、session token、assume-role provider 不一致。P0 处理： + +- Directory Bucket 直接返回明确 NotSupported。 +- 普通 S3 保持现状。 +- 后续开放前必须让 presigner 使用统一 credentials provider，并验证 URL 的签名类型、bucket host、session token、最大有效期和 SDK 刷新行为。 + +### 10.6 Hadoop/S3A 和 Java v1 + +对所有 External Catalog、Hadoop FileSystem、AWS Java SDK v1 client 做调用面清单。这份清单是 P0 的合入产物和阻断门禁，至少逐项记录：用户入口、最终 I/O 实现及依赖版本、何时能取得完整 bucket、最早可确定拒绝的位置、负责人和自动化测试。初始清单至少覆盖： + +| 调用面家族 | 已知入口示例 | P0 处理 | +| --- | --- | --- | +| Doris 原生 S3 FileSystem | S3ObjStorage、S3FileSystem、S3Util | 按本设计适配并支持 | +| Hive BE-upload/FE-complete | VHivePartitionWriter、HMSTransaction、ObjFileSystem | 按第 8.5 节传播 multipart checksum 后支持 | +| Hudi Hadoop FileSystem | HudiScanNode 及其 Hadoop 配置构造 | 未证明所装载 S3A 版本支持前拒绝 | +| LakeSoul Hadoop FileSystem | LakeSoulUtils 及其 Hadoop 配置构造 | 未证明所装载 S3A 版本支持前拒绝 | +| Paimon/Hadoop 配置 | AbstractPaimonProperties 及对应 catalog/scanner | 未证明最终 I/O client 支持前拒绝 | +| be-java-extensions 与其他外部 catalog/scanner | 各扩展自己的 classloader 和 AWS/Hadoop 依赖 | 逐模块锁定实际依赖版本；未验证项拒绝 | +| AWS Java SDK v1 直接客户端 | fe-core 及扩展中引入 aws-java-sdk-s3 的路径 | P0 不支持，迁移或隔离到 Java SDK v2 后再开放 | + +P0 规则： + +- 若最终 I/O 由已适配的 Doris S3 FileSystem 执行，允许。 +- 若最终 I/O 由未经验证的 S3A 或 Java v1 client 执行，在能够取得完整 bucket 的最早确定点拒绝 Directory Bucket URI。bucket 在分析期可知时就在分析期拒绝；只能在运行时 URI 中取得时，必须在 filesystem/scanner factory 发出首个网络请求前拒绝。 +- 错误必须指出该 surface 尚未支持，不能在运行时表现为 403、错误 endpoint 或签名失败。 +- 每一行清单都必须有正向普通 S3 用例和 Directory Bucket 负例；清单未穷举或缺少测试时，PR 3 不满足完成条件。 + +## 11. 配置与用户使用 + +### 11.1 推荐配置 + +Directory Bucket 已由管理员在 us-east-1 的 use1-az4 创建： + + analytics-hot--use1-az4--x-s3 + +Doris S3 Resource 示例： + + CREATE RESOURCE "s3_express_hot" + PROPERTIES + ( + "type" = "s3", + "provider" = "S3", + "AWS_ENDPOINT" = "https://s3.us-east-1.amazonaws.com", + "AWS_REGION" = "us-east-1", + "AWS_BUCKET" = "analytics-hot--use1-az4--x-s3", + "AWS_ROOT_PATH" = "hot-data/", + "AWS_ACCESS_KEY" = "", + "AWS_SECRET_KEY" = "", + "use_path_style" = "false" + ); + +对象 URI： + + s3://analytics-hot--use1-az4--x-s3/hot-data/example.parquet + +优先使用 instance profile、web identity 或 assume role，避免在 Resource 中保存静态 AK/SK。上例只用于展示已有属性形状。 + +Doris 当前 S3 属性的合法 provider 值是 S3，不是 AWS。是否连接官方 AWS 服务由 endpoint authority 和 override mode 判断；不要为了本功能新增一个只在部分链路存在的 AWS 枚举。 + +### 11.2 endpoint 规则 + +- 用户不需要手工配置 bucket-specific Zonal endpoint。 +- 如果当前 Doris surface 强制要求 AWS_ENDPOINT，填写标准 regional S3 HTTPS endpoint。 +- Directory Bucket client 不把该值传给 endpointOverride。 +- 普通 bucket 继续保留显式 regional、FIPS、dualstack 和 S3-compatible endpoint 行为；P0 的 Directory Bucket 只接受 AWS_SDK_ROUTED，遇到 FIPS、dualstack 或其他显式 override 配置时拒绝，而不是静默改写。 +- s3express-control endpoint 不能用于对象 I/O。 +- 自定义 proxy 若会改写 Host、DNS 或 TLS SNI，不在支持范围。 + +### 11.3 部署要求 + +- 计算节点和 bucket 位于同一 AZ 才能获得设计目标中的低延迟和更低网络成本。 +- 跨 AZ 功能上可能可达，但不作为性能目标，且会增加成本和故障域复杂度。 +- 私网访问使用 com.amazonaws..s3express 的 gateway VPC endpoint；普通 S3 endpoint 或 PrivateLink 配置不能直接替代。 +- DNS、TLS SNI 和 Host header 必须保留 SDK 生成的 bucket-specific host。 + +## 12. Cloud Storage Vault 边界 + +### 12.1 为什么 P0 不支持 + +Cloud Storage Vault 是 Doris Cloud 的主持久化存储，不是临时 staging 目录。Directory Bucket： + +- 只在单个 AZ 内冗余。 +- 不支持 Versioning。 +- 删除后没有普通 S3 versioning 提供的恢复窗口。 + +当前 Cloud checker 还明确依赖： + +- S3Accessor::check_versioning() 要求 Versioning=Enabled。 +- get_life_cycle() 查找 NoncurrentVersionExpiration。 +- Checker 使用该保留天数做巡检间隔风险判断。 + +Directory Bucket 不支持这两个前提。简单地“跳过检查”会移除现有安全门禁；把 current-object Expiration 当成替代又可能删除仍然有效的 Doris 数据。因此 P0 必须在创建或修改 Storage Vault 时拒绝 Directory Bucket，而不是让 BE 写入成功后由 recycler/checker 失败。 + +只在 FE 校验不够：cloud/src/common/http_helper.cpp 暴露 add_s3_vault、alter_s3_vault 等直接 HTTP 路由，它们进入 cloud/src/meta-service/meta_service_resource.cpp 并可独立持久化 ObjectStoreInfoPB。P0 要在两层都执行同一 service-aware 门禁： + +1. FE 在生成 Storage Vault 请求前拒绝 official AWS service + 合法 --x-s3 bucket。 +2. Meta Service 在 ADD_S3_VAULT、ALTER_S3_VAULT 以及所有能直接创建或替换 S3 vault/object info 的 RPC/HTTP 路径中，必须在 txn->put 或修改 InstanceInfoPB 前再次拒绝。 +3. Meta Service 的判断必须同时使用 provider/service、endpoint authority/override mode 和完整 bucket；第三方 S3-compatible 服务中恰好以 --x-s3 结尾的 bucket 不能被误拒绝。 +4. 返回稳定的 INVALID_ARGUMENT/UnsupportedS3Surface 错误，不能先落库再依赖 recycler 或 checker 报错。 + +这是一项产品范围门禁，不改变现有 Cloud 信任边界或安全定级。 + +### 12.2 若产品决定支持 Cloud Vault + +必须单独通过以下设计门禁后才能进入 P1： + +1. 产品和运维明确接受单 AZ 与无 versioning 的数据耐久性、恢复和删除风险。 +2. cloud/src/recycler/s3_accessor.cpp 改用 S3ClientConfiguration、S3EndpointProvider、SDK endpoint rules、HTTPS 和 virtual-hosted。 +3. S3Conf::from_obj_store_info 当前把 OSS、S3、COS、OBS、BOS 都折叠成 S3Conf::S3；P1 必须保留原始 ObjectStoreInfoPB::Provider 或在折叠前派生 official AWS service kind，不能把第三方 vault 的 --x-s3 bucket 误判为 Directory Bucket。 +4. cloud/src/recycler/s3_obj_client.cpp 的 Put、DeleteObjects、List、Abort 使用 Directory Bucket checksum 和分页语义。 +5. 不把 session credentials 写入 ObjectStoreInfoPB 或 Meta Service。 +6. client/session cache 不能跨 instance、tenant、vault 或 credentials identity 共享。 +7. 重做 checker 的安全模型，不能仅 skip versioning/lifecycle： + - 明确 Directory Bucket 下不可恢复删除的告警和运维门禁。 + - lifecycle 只允许验证 AbortIncompleteMultipartUpload，不允许把通用 current-object Expiration 配在 Doris 数据根目录。 + - 为 checker 设计不依赖 noncurrent version retention 的独立巡检 SLA。 +8. recycler 的 mark-before-delete、两阶段删除、重试幂等和 packed-file 顺序保持不变。 +9. 完成 Cloud recycler、checker、snapshot、restore、storage vault 创建和滚动升级的端到端测试。 + +在这套安全模型评审完成前，Cloud 代码可以先复用 capability resolver 用于 fail fast，但不对外开放 Vault。 + +## 13. 安全设计 + +### 13.1 信任边界 + +按照 Doris threat model，外部 S3 配置由管理员信任，但 credentials、网络 endpoint 和 Cloud tenant 隔离仍属于必须保护的边界。 + +### 13.2 Credentials + +- 长期 AK/SK、AWS_TOKEN、role ARN 等继续走 Doris 现有安全配置和 credentials provider。 +- CreateSession 返回值只存在于 SDK 进程内存。 +- 不新增 session token 的 FE 属性、PB、Thrift、EditLog 或 Meta Service 字段。 +- 不记录 Authorization、x-amz-s3session-token、secret key、session token。 +- 修复 S3ClientConf::to_string() 对 token 的明文输出；external ID 也按敏感信息处理。 +- 日志可记录 bucket mode、region、zone-id、operation、HTTP status、AWS request id，不记录完整 credentials。 + +### 13.3 网络 + +- Directory Bucket 强制 HTTPS。 +- 不接受 path-style 或任意 endpoint override。 +- 保留 Doris 现有 endpoint 安全校验；官方 AWS host 仍需经过 allowlist/SSRF 规则。 +- 不跟随会把签名请求重定向到非 AWS host 的重定向。 + +### 13.4 Cloud tenant 隔离 + +若未来支持 Cloud： + +- cache key 包含 instance/vault/credentials identity。 +- 一个 tenant 的 session client 不得用于另一个 tenant，即使 bucket、region 和 endpoint 相同。 +- session 刷新失败只影响对应 client，不触发全局 credentials 降级。 + +## 14. 错误处理 + +建议新增或规范化以下用户可识别错误： + +| 错误 | 示例信息 | +| --- | --- | +| 非法 bucket 名 | AWS Directory Bucket requires a full bucket name ending with --x-s3 | +| 缺少 region | AWS Directory Bucket requires AWS_REGION | +| path-style | Path-style addressing is not supported for AWS Directory Bucket | +| HTTP | AWS Directory Bucket requires HTTPS | +| endpoint 冲突 | Configured endpoint region/zone does not match directory bucket | +| control endpoint | s3express-control endpoint cannot be used for object operations | +| session 权限 | Access denied while creating S3 Express session; grant s3express:CreateSession | +| 不支持的 surface | S3 Express One Zone is not supported by Hadoop S3A/Cloud Storage Vault in this release | +| checksum | CRC32C mismatch for UploadPart, bucket=..., key=..., part=... | +| multipart 空洞 | Directory Bucket multipart parts must be consecutive from 1 | +| list 资源不足 | Directory Bucket listing exceeded query memory limit while preserving ordered result | + +实现遵循 Doris 的“错误即失败”原则： + +- 配置不一致在创建 client 前失败。 +- checksum 不一致不能重试为无 checksum 请求。 +- session auth 失败不能回退为匿名或非 TLS。 +- Abort 失败不能被静默丢弃。 +- 只对 SDK 明确标记可重试的网络、限流和服务端错误使用现有 retry strategy。 + +## 15. 可观测性 + +### 15.1 日志 + +client 创建时记录： + +- bucket_type=general_purpose 或 directory +- service kind（official AWS 或 custom S3-compatible） +- region +- zone_id +- endpoint_mode=override 或 sdk_rules +- addressing=virtual 或 path +- checksum_policy + +请求失败记录： + +- operation +- bucket/key 的现有安全形式 +- HTTP status +- SDK exception name +- AWS request id +- 是否处于 Directory Bucket 模式 + +不在 INFO 级别记录每次成功请求或 session token。 + +### 15.2 Metrics + +建议在现有 S3 bvar 基础上增加低基数维度： + +- s3_client_create_total，按 bucket_type +- s3_request_total，按 bucket_type、operation、result +- s3_request_latency，按 bucket_type、operation +- s3_checksum_failure_total，按 operation +- s3_multipart_abort_total，按 result +- s3_directory_list_scanned_keys +- s3_directory_list_returned_keys +- s3_directory_list_pages +- s3_auth_failure_total，按 AccessDenied、ExpiredToken、其他 + +不要把 bucket 名、key 或 tenant id 作为 metric label。 + +AWS SDK 内部 CreateSession 不一定暴露独立回调；若不能可靠计数，不应猜测 session refresh 次数。可以通过 SDK debug 日志和长时集成测试验证刷新，通过外层 403/ExpiredToken 统计定位故障。 + +## 16. 测试方案 + +### 16.1 Capability resolver 单测 + +C++ 与 Java 使用同一组测试向量： + +- AWS + 合法 --x-s3。 +- bucket 中间包含 --x-s3 但后缀不匹配。 +- 自定义 S3-compatible endpoint + --x-s3。 +- AWS + custom endpoint。 +- zonal endpoint 的 region/zone 匹配与不匹配。 +- control endpoint。 +- 空 region。 +- HTTP。 +- path-style。 +- dualstack 官方 endpoint。 + +### 16.2 BE client 配置单测 + +通过可注入 builder 或 request hook 断言： + +- Directory Bucket：HTTPS、virtual、无 endpointOverride、disableS3ExpressAuth=false。 +- 普通 AWS S3：原 endpoint 和 addressing 行为不变。 +- S3-compatible：保留 endpointOverride，Express auth 禁用。 +- CA、aws_client_request_timeout_ms、resource timeout、connect timeout、max connections、retry strategy 被保留。 +- payload signing policy 与设计一致。 +- cache key 能隔离 need_override_endpoint、bucket type、scheme、checksum policy 和 credentials identity。 + +### 16.3 Checksum 单测 + +- CRC32C 已知向量：空字符串、123456789、小 buffer、跨 chunk buffer。 +- PutObject request： + - Directory 有 ChecksumAlgorithm=CRC32C 和正确 ChecksumCRC32C。 + - Directory 无 ContentMD5。 + - 普通 S3 仍有原 ContentMD5。 +- DeleteObjects request body 使用 CRC32C。 +- checksum mismatch 返回失败。 + +### 16.4 Multipart 单测 + +- Create 声明 CRC32C。 +- 多个并发 part 的 checksum 正确保存。 +- Complete 前排序。 +- Complete 携带每个 part 的 ETag 和 ChecksumCRC32C。 +- Hive BE-upload/FE-complete 路径把 checksum algorithm 和 part checksum 通过 TS3MPUPendingUpload 传到 FE。 +- Directory Bucket 下缺少可选 checksum 字段时拒绝 Complete 并 Abort。 +- part number 从 1 连续。 +- 空洞、重复、从 0 开始均失败。 +- UploadPart 失败时等待其他 future 后 Abort。 +- Complete 失败时 Abort。 +- Abort 的 NoSuchUpload 幂等成功。 +- 原始失败与 Abort 失败同时可见。 +- 普通 S3 和 Azure writer 不回归。 + +### 16.5 List 单测 + +构造无序、多页、超过 1000 个 key 的 mock 响应： + +- 不发送 StartAfter。 +- 服务 prefix 以 / 结尾。 +- 正确使用 NextContinuationToken。 +- 对 originalPrefix 和 glob 二次过滤。 +- 无序输入得到稳定有序输出。 +- startAfter 在本地应用。 +- 仅 maxFiles 时最多保留 N+1 个全局字典序最小对象，并生成正确 maxFile。 +- maxBytes 时先完整排序，包含触发阈值的对象，并生成正确 maxFile。 +- 未完成 multipart prefix 不被当成对象。 +- 空页但 IsTruncated=true 时继续翻页。 +- 递归删除每批删除后从空 token 重扫，不复用删除前的 opaque continuation token。 +- 前几页没有匹配 key 时继续扫描，不能误判删除完成。 +- 取消和内存不足返回错误。 + +### 16.6 FE 单测 + +- properties + URI 得到与 C++ 一致的 capability。 +- AWS_SDK_ROUTED 的 directory client 不 endpointOverride、不强制 signer、Express auth enabled；Directory request 在 path-style=true 时 fail fast。 +- general client 保留普通 AWS FIPS/dualstack/显式 endpoint 与 S3-compatible 既有行为。 +- AWS_EXPLICIT_ENDPOINT + Directory Bucket fail fast。 +- bucket property 为空时，从每次 remotePath 派生 capability；同一 S3ObjStorage 的 general/directory 两个 lazy client 不串配置。 +- custom endpoint 下以 --x-s3 结尾的 bucket 仍按 GENERAL_PURPOSE。 +- default chain、static session token、web identity、assume role provider 能传给 client。 +- Java PutObject、CreateMultipartUpload、UploadPart、CompleteMultipartUpload 的 CRC32C request/response 字段完整。 +- HMSTransaction 和 ObjFileSystem 正确消费 TS3MPUPendingUpload 的 checksum 字段；旧消息、缺字段和双失败路径有负例。 +- S3Resource ping 使用正确 Head/List 请求。 +- S3Resource ping 的业务失败与 Abort 失败同时可见，覆盖双失败测试。 +- Directory Bucket presign 返回 NotSupported。 +- 第 10.6 节调用面清单中的每一个入口都在“最早能取得完整 bucket 且尚未发出网络请求”的位置覆盖 Directory Bucket 负例，并保留普通 S3 正例；不能统一用一个抽象测试代替各入口门禁。 + +### 16.7 Cloud P0 门禁单测 + +- FE 创建和修改 Storage Vault 时拒绝 official AWS Directory Bucket。 +- 绕过 FE，直接调用 Meta Service ADD_S3_VAULT 和 ALTER_S3_VAULT 时，在任何持久化写入前返回 INVALID_ARGUMENT。 +- 覆盖其他能够直接创建或替换 S3 vault/object info 的 RPC/HTTP 路径，确认无法绕过同一门禁。 +- custom S3-compatible endpoint + --x-s3 后缀 bucket 不误拒绝。 +- 拒绝后 InstanceInfoPB、StorageVaultPB 和 recycler 可见状态均无变化。 + +### 16.8 AWS 真实集成测试 + +使用预创建的 Directory Bucket，凭证和 bucket 名通过 CI secret/environment 注入；测试默认关闭，只在专用 AWS job 运行。 + +必测： + +1. Put/Get/Head/Delete 小对象和空对象。 +2. 大对象 multipart，读取后逐字节校验。 +3. List 超过 1000 个 key、无序返回、glob、startAfter、maxFiles、maxBytes 和 maxFile。 +4. DeleteObjects。 +5. 主动 Abort 后确认 upload 不存在。 +6. 静态 IAM credentials。 +7. AssumeRole 或 web identity。 +8. 只读 policy 的读成功、写失败。 +9. 缺少 s3express:CreateSession 的明确 403。 +10. client 连续运行超过六分钟并跨越 session refresh。 +11. 并发首次请求，验证无 session refresh stampede 和请求失败。 +12. 错 region、HTTP、path-style、control endpoint 的负例。 +13. 同 AZ 和跨 AZ 只记录延迟差异，不设置易抖动的硬性能阈值。 + +普通 S3 和 MinIO 只能证明兼容性，不能替代 CreateSession 和 Directory Bucket 语义测试。 + +### 16.9 回归与工程检查 + +按改动阶段执行： + +- ./build.sh --be --fe +- run-be-ut.sh 运行相关 s3_util、s3_obj_storage_client、s3_file_writer 测试 +- run-fe-ut.sh 运行 S3URI、S3Properties、S3Util、S3ObjStorage 测试 +- run-regression-test.sh 运行 S3 Resource、TVF、外部文件读写相关 case +- build-support/clang-format.sh 格式化修改的 C++ 文件 +- build-support/check-format.sh 检查格式 +- BE build 生成 compile_commands.json 后，对修改文件运行 build-support/run-clang-tidy.sh + +若没有 Directory Bucket AWS job 通过，不得把 feature 标记为 GA。 + +## 17. 兼容性与滚动升级 + +### 17.1 既有工作负载 + +- 不改变普通 AWS S3 和 S3-compatible 的属性名称。 +- 不改变对象 key 或文件格式。 +- bucket type 不新增持久化字段；TS3MPUPendingUpload 只增加向后可解析的可选 checksum 字段。 +- 删除当前全局 s3_disable_content_md5，不让其改变既有 cached client。 +- 普通 S3 继续使用既有 Content-MD5 和 endpoint override 策略。 + +### 17.2 混合版本 + +旧 BE 无法可靠访问 Directory Bucket。滚动升级规则： + +1. 可以在混合版本期间继续使用普通 S3。 +2. 必须升级所有可能执行该资源 I/O 的 BE 和 FE 后，才能创建或启用 Directory Bucket Resource。 +3. TS3MPUPendingUpload 的新字段虽然是 optional，但旧 FE 会忽略 part checksum，旧 BE 也不会生成它；因此 optional 只保证协议解析，不保证 Directory Bucket 功能兼容。 +4. 若未来开放 Cloud Vault，所有 recycler/checker 实例必须先升级。 +5. 回滚到不支持版本前必须停止 Directory Bucket workload，并按第 20 节先把仍需访问的数据复制到普通 S3、切换到新资源；对象内容无需改格式，但旧版本不能读取留在 Directory Bucket 中的数据。 + +可选增强是增加 cluster capability bit，让 FE 在所有 BE 尚未声明 S3_DIRECTORY_BUCKET 时拒绝启用。若 Doris 当前没有适合的能力协商机制，首期通过发布文档和运维检查实现，不为此引入新的持久化协议。 + +## 18. 分阶段交付 + +### PR 1：SDK 基线与公共能力模型 + +内容： + +- 固定 AWS SDK C++ 1.11.400，并修正文档中的版本根因。 +- 保留 aws_logger vaLog 适配。 +- 增加 service-aware capability resolver 和跨语言测试向量。 +- 修复敏感 token 日志。 +- client config 从现有公共配置转换，补齐 cache key。 + +完成条件： + +- 普通 S3、MinIO 配置 UT 通过。 +- Directory Bucket client 配置 UT 通过。 +- 无业务 API 行为声明。 + +### PR 2：BE 读写完整链路 + +内容： + +- Directory Bucket endpoint/auth client。 +- PutObject CRC32C。 +- multipart CRC32C 全链路。 +- Hive BE-upload/FE-complete 的可选 Thrift checksum 传播。 +- AbortMultipartUpload。 +- DeleteObjects checksum。 +- exact-key 读写。 + +完成条件： + +- BE UT、真实 AWS 小对象和 multipart job 通过。 +- session refresh 超过六分钟通过。 +- 普通 S3、MinIO 回归通过。 + +### PR 3：List 与 FE + +内容： + +- S3CompatibleFileSystem 和 S3FileSystem 的 continuation token + 本地过滤排序算法。 +- S3ObjStorage、S3FileSystemProperties、S3Util、S3Resource builder 和 ping。 +- 完成第 10.6 节所有 Hadoop/S3A、Java v1、external catalog/scanner surface 清单，并逐入口 fail fast。 +- FE 与 Meta Service 在所有 Storage Vault 创建、修改入口执行 P0 fail fast。 +- Presigned URL 门禁。 + +完成条件： + +- 多页无序 List、glob、startAfter、maxFiles、maxBytes、maxFile 语义测试通过。 +- 调用面清单没有“待盘点”项，每个未支持入口都有普通 S3 正例和 Directory Bucket 负例。 +- 绕过 FE 直调 Meta Service 的 add/alter vault 测试证明拒绝发生在持久化前，且第三方 --x-s3 bucket 不误判。 +- FE UT、regression、真实 AWS 列举通过。 + +### PR 4：发布、文档与可观测性 + +内容： + +- 用户文档、IAM 示例、部署限制。 +- metrics 和错误信息。 +- mixed-version 与 rollback 文档。 +- AWS 专用 CI job 稳定运行。 + +完成条件： + +- P0 验收清单全部通过。 +- 发布说明明确 Cloud Vault、S3A、presign 等边界。 + +### 独立 P1：Cloud Storage Vault + +只有第 12 节的安全模型通过产品、Cloud 和运维评审后才启动，不能作为 P0 PR 的顺手补丁。 + +## 19. P0 验收标准 + +以下条件必须全部满足： + +1. 官方 AWS service + 合法 --x-s3 bucket 被稳定分类为 Directory Bucket；第三方兼容存储不误判。 +2. 线上请求使用 SDK 生成的 bucket-specific Zonal HTTPS virtual-host，不出现 path-style 或自定义 endpointOverride。 +3. Doris 未显式调用 CreateSession，但 AWS SDK 能自动创建、复用并刷新 session。 +4. client 连续运行跨越至少一次五分钟 session 过期仍能无中断读写。 +5. PutObject、UploadPart、CompleteMultipartUpload 和 DeleteObjects checksum 行为符合设计。 +6. multipart 任一失败会主动 Abort，不留下由正常失败路径产生的未完成 upload。 +7. ListObjectsV2 不发送 StartAfter，不依赖服务排序，多页结果符合 Doris 现有排序、startAfter、maxFiles、maxBytes 和 maxFile 契约。 +8. 普通 AWS S3、MinIO/S3-compatible、Azure writer 的现有测试不回归。 +9. FE 连通性检查使用相同 bucket 分类和 credentials provider。 +10. 第 10.6 节清单中的 Cloud Vault、S3A、Java v1、presign 等未支持路径都在最早确定点、首个网络请求或持久化之前 fail fast；FE 与 Meta Service 的直接 add/alter Vault 路径均有证据，第三方兼容存储不误判。 +11. 日志、metrics、PB/Thrift 中不存在 session secret 或明文 token。 +12. C++ format、clang-tidy、BE UT、FE UT、regression 和专用 AWS 集成测试通过。 +13. 用户文档明确单 AZ 故障域、同 AZ 部署建议、IAM、endpoint 和支持边界。 + +## 20. 回滚方案 + +- feature 不改变对象内容格式，但这不等于可以直接回滚：不支持该功能的旧 Doris binary 无法读取仍只存在于 Directory Bucket 的数据。 +- “停止流量/关闭新写入”和“恢复旧版本下的数据可用性”是两个不同层次。前者可以立即执行；后者必须完成数据复制和资源迁移后才能执行。 +- 运行时关闭方式是停止使用 Directory Bucket Resource，而不是动态切换 checksum 或禁用 session auth。 +- 为需要快速回滚的生产 workload，启用前应预先创建普通 S3 bucket、复制账号/权限、容量和校验清单；必要时在切换前安排持续复制或业务双写。P0 不在 Doris 内实现自动双写。 +- 若新版本出现 session、endpoint 或 list 问题，按以下顺序回滚： + 1. 停止所有新的 Directory Bucket 写入和会创建对象的作业，记录切流时间与最后成功提交点。 + 2. 等待正在进行的 multipart 完成；不能完成的 upload 使用仍受支持的新版本主动 Abort，并确认没有遗留 upload。 + 3. 在新版本 Doris/SDK 仍可访问 Directory Bucket 时，使用经过批准的外部复制工具或 Doris 导出路径，把仍需访问的对象复制到预建的普通 S3 bucket/root。保持对象 key，使用对象数量、字节数、清单和 checksum/逐对象读取校验复制完整性。 + 4. 新建指向普通 S3 bucket/root 的 S3 Resource 或 Storage Policy；不要试图原地修改旧 S3Resource 的 endpoint、region、root path 或 bucket，这些属性在当前实现中不可修改。 + 5. 使用 Doris 已支持的资源/策略/表迁移流程把 workload 切到新资源，执行代表性查询、写入和恢复检查；只有确认不再引用 Directory Bucket 后，才回滚 binary。 + 6. 保留 Directory Bucket 为只读观察窗口，达到既定保留期且清单复核完成后再删除数据。 +- P0 不支持 Cloud Storage Vault，因此上述切换只适用于本设计支持的外部 S3 Resource/FileSystem workload；不能把它解读为 Cloud 主存储的透明迁移方案。 +- 如果故障发生时没有可用的新版本读取路径，也没有预先复制的数据，只能停止流量，不能宣称完成了数据可用性回滚。 +- 不允许在同一个 Directory Bucket 上通过强制 endpointOverride 或 disableS3ExpressAuth 临时绕过故障，因为这可能产生错误签名、错误 host 或无效 checksum。 + +## 21. 待确认事项 + +下面事项不改变本文的 P0 架构决策，但必须在对应实现 PR 进入合并前关闭： + +1. SDK C++ 1.11.400 对 composite CRC32C 的具体 request/response model 字段，以编译和真实 AWS 测试为准。 +2. Java SDK v2 当前 pinned 版本在 Doris classloader 场景下的 S3 Express interceptor 加载行为。 +3. 第 10.6 节完整调用面清单、实际依赖版本和逐入口测试属于 PR 3 的阻断项，不能以“后续盘点”状态合入。 +4. presigned URL 是否有明确业务需求；若有，另写设计。 +5. Cloud Storage Vault 是否允许单 AZ、无 versioning 存储作为主数据层；默认答案为不允许。 +6. 是否增加 cluster capability bit 阻止混合版本启用 Directory Bucket。 + +## 22. 官方资料与源码依据 + +AWS： + +- [Differences for directory buckets](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-differences.html) +- [Directory bucket API operations](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-APIs.html) +- [Creating directory buckets in an Availability Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-create.html) +- [Networking for directory buckets in an Availability Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-az-networking.html) +- [S3 Express One Zone session authentication](https://docs.aws.amazon.com/sdkref/latest/guide/feature-s3-express.html) +- [Authenticating and authorizing requests](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-authenticating-authorizing.html) +- [CreateSession API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html) +- [PutObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) +- [DeleteObjects API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) +- [ListObjectsV2 API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) +- [Using multipart uploads with directory buckets](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-using-multipart-upload.html) +- [PutBucketLifecycleConfiguration API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) +- [Optimizing directory bucket performance](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-optimizing-performance.html) + +AWS SDK C++： + +- [AWS SDK C++ maintainer discussion: S3 Express customizations merged in 1.11.212](https://github.com/aws/aws-sdk-cpp/discussions/2623) +- [AWS SDK C++ 1.11.219 source tree](https://github.com/aws/aws-sdk-cpp/tree/1.11.219) +- [AWS SDK C++ S3ClientConfiguration](https://docs.aws.amazon.com/sdk-for-cpp/latest/api/aws-cpp-sdk-s3/html/struct_aws_1_1_s3_1_1_s3_client_configuration.html) +- [AWS SDK Java v2 S3BaseClientBuilder](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3BaseClientBuilder.html) + +Doris 当前实现入口： + +- be/src/util/s3_util.cpp +- be/src/util/s3_util.h +- be/src/io/fs/s3_obj_storage_client.cpp +- be/src/io/fs/s3_file_writer.cpp +- be/src/exec/sink/writer/vhive_partition_writer.cpp +- gensrc/thrift/DataSinks.thrift +- fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +- fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +- fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java +- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/FileSystem.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjStorage.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectListOptions.java +- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java +- fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java +- fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +- fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulUtils.java +- fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +- cloud/src/common/http_helper.cpp +- cloud/src/meta-service/meta_service_resource.cpp +- cloud/src/recycler/s3_accessor.cpp +- cloud/src/recycler/s3_obj_client.cpp +- cloud/src/recycler/checker.cpp From 8f732dce8a00eb911e1e97bc6523076b79098cc2 Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 16 Jul 2026 23:23:23 +0800 Subject: [PATCH 05/13] [feature](s3) Support S3 Express One Zone directory buckets ### What problem does this PR solve? Issue Number: None Related PR: #63409 Problem Summary: Add end-to-end S3 Express One Zone Directory Bucket support across native BE and FE S3 paths. Use SDK-managed session authentication and zonal endpoint routing, CRC32C multipart propagation, Directory Bucket listing semantics, active multipart cleanup, and fail-fast gates for unsupported Cloud Storage Vault and Hadoop/S3A surfaces while preserving ordinary S3-compatible behavior. ### Release note Add native S3 Express One Zone Directory Bucket support for Doris S3 file access. Cloud Storage Vault, Hadoop/S3A-backed external surfaces, and presigned URLs remain unsupported and fail fast. ### Check List (For Author) - Test: Test code added but not run per request - Behavior changed: Yes. Official AWS Directory Buckets use SDK-managed Express session authentication, CRC32C, and Directory Bucket listing semantics; unsupported surfaces fail fast. - Does this need documentation: Yes, goal.md --- be/src/common/config.cpp | 2 - be/src/common/config.h | 6 - .../sink/writer/vhive_partition_writer.cpp | 11 + be/src/exprs/function/ai/embed.h | 15 +- be/src/io/fs/obj_storage_client.h | 7 + be/src/io/fs/s3_file_writer.cpp | 106 ++++--- be/src/io/fs/s3_obj_storage_client.cpp | 150 ++++++++-- be/src/io/fs/s3_obj_storage_client.h | 9 +- be/src/io/tools/file_cache_microbench.cpp | 1 + be/src/util/s3_util.cpp | 93 ++++-- be/src/util/s3_util.h | 56 ++-- be/test/io/fs/s3_file_writer_test.cpp | 29 ++ be/test/util/s3_util_test.cpp | 42 ++- .../meta-service/meta_service_resource.cpp | 27 ++ common/cpp/s3_bucket_capabilities.h | 200 +++++++++++++ .../org/apache/doris/avro/S3FileReader.java | 4 + .../java/org/apache/doris/avro/S3Utils.java | 30 ++ .../org/apache/doris/avro/S3UtilsTest.java | 36 +++ fe/fe-connector/fe-connector-hive/pom.xml | 6 + .../connector/hive/HiveScanPlanProvider.java | 13 + .../hive/HiveScanPlanProviderTest.java | 43 +++ .../org/apache/doris/catalog/S3Resource.java | 39 ++- .../apache/doris/catalog/StorageVaultMgr.java | 7 + .../org/apache/doris/common/util/S3URI.java | 27 +- .../org/apache/doris/common/util/S3Util.java | 108 ++----- ...bstractS3CompatibleConnectivityTester.java | 2 +- .../doris/datasource/hive/HMSTransaction.java | 12 +- .../datasource/hudi/source/HudiScanNode.java | 12 +- .../lakesoul/source/LakeSoulScanNode.java | 9 + .../metastore/AbstractPaimonProperties.java | 15 + .../PaimonAliyunDLFMetaStoreProperties.java | 1 + .../PaimonFileSystemMetaStoreProperties.java | 1 + .../PaimonHMSMetaStoreProperties.java | 1 + .../PaimonJdbcMetaStoreProperties.java | 1 + .../PaimonRestMetaStoreProperties.java | 1 + .../filesystem/S3BucketCapabilities.java | 277 ++++++++++++++++++ .../doris/filesystem/UploadPartResult.java | 10 + .../filesystem/S3BucketCapabilitiesTest.java | 93 ++++++ .../doris/filesystem/hdfs/DFSFileSystem.java | 8 + .../filesystem/hdfs/DFSFileSystemTest.java | 23 ++ .../doris/filesystem/s3/S3FileSystem.java | 88 ++++-- .../filesystem/s3/S3FileSystemProperties.java | 12 +- .../doris/filesystem/s3/S3ObjStorage.java | 229 +++++++++++---- .../s3/S3FileSystemPropertiesTest.java | 14 + .../doris/filesystem/s3/S3FileSystemTest.java | 25 +- .../filesystem/s3/S3ObjStorageMockTest.java | 87 ++++++ .../doris/filesystem/spi/ObjFileSystem.java | 12 +- .../spi/S3CompatibleFileSystem.java | 19 +- gensrc/thrift/DataSinks.thrift | 6 + 49 files changed, 1709 insertions(+), 316 deletions(-) create mode 100644 common/cpp/s3_bucket_capabilities.h create mode 100644 fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java create mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java create mode 100644 fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java create mode 100644 fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 5fa1fa6091973d..9ec587bd100ec5 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1512,8 +1512,6 @@ DEFINE_mInt32(s3_read_max_wait_time_ms, "800"); DEFINE_mBool(enable_s3_object_check_after_upload, "true"); DEFINE_mInt32(aws_client_request_timeout_ms, "30000"); -DEFINE_mBool(s3_disable_content_md5, "false"); - DEFINE_mBool(enable_s3_rate_limiter, "false"); DEFINE_mInt64(s3_get_bucket_tokens, "1000000000000000000"); DEFINE_Validator(s3_get_bucket_tokens, [](int64_t config) -> bool { return config > 0; }); diff --git a/be/src/common/config.h b/be/src/common/config.h index c9a6241b45882f..012f09a735c0cd 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1621,12 +1621,6 @@ DECLARE_mInt32(s3_read_max_wait_time_ms); DECLARE_mBool(enable_s3_object_check_after_upload); DECLARE_mInt32(aws_client_request_timeout_ms); -// When true, omit the Content-MD5 header on S3 PutObject / UploadPart and send a -// CRC32C checksum instead. Required for S3 Express One Zone, which returns 501 -// NotImplemented for Content-MD5. Endpoints containing "s3express" auto-enable -// this regardless of the flag. -DECLARE_mBool(s3_disable_content_md5); - // write as inverted index tmp directory DECLARE_String(tmp_file_dir); diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 8331efac54bd47..2f4bf8f920b156 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -19,6 +19,7 @@ #include +#include "common/check.h" #include "core/block/materialize_block.h" #include "core/column/column_map.h" #include "format/transformer/vcsv_transformer.h" @@ -197,10 +198,20 @@ bool VHivePartitionWriter::_build_s3_mpu_pending_upload(TS3MPUPendingUpload* pen pending_upload->__set_upload_id(upload_id); std::map etags; + std::map part_checksums; for (auto& completed_part : s3_mpu_file_writer->completed_parts()) { etags.insert({completed_part.part_num, completed_part.etag}); + if (completed_part.checksum_crc32c.has_value()) { + part_checksums.insert( + {completed_part.part_num, *completed_part.checksum_crc32c}); + } } pending_upload->__set_etags(etags); + if (!part_checksums.empty()) { + DORIS_CHECK_EQ(part_checksums.size(), etags.size()); + pending_upload->__set_checksum_algorithm(TObjectStorageChecksumAlgorithm::CRC32C); + pending_upload->__set_part_checksums(part_checksums); + } return true; } diff --git a/be/src/exprs/function/ai/embed.h b/be/src/exprs/function/ai/embed.h index 2367e4b9459540..74758c251df08a 100644 --- a/be/src/exprs/function/ai/embed.h +++ b/be/src/exprs/function/ai/embed.h @@ -374,16 +374,21 @@ class FunctionEmbed : public AIFunction { S3ClientConf s3_client_conf; RETURN_IF_ERROR(init_s3_client_conf_from_json(file_input, s3_client_conf)); - auto s3_client = S3ClientFactory::instance().create(s3_client_conf); - if (s3_client == nullptr) { - return Status::InternalError("Failed to create S3 client for EMBED file input"); - } - S3URI s3_uri(uri); RETURN_IF_ERROR(s3_uri.parse()); std::string bucket = s3_uri.get_bucket(); std::string key = s3_uri.get_key(); DORIS_CHECK(!bucket.empty() && !key.empty()); + s3_client_conf.bucket = bucket; + if (resolve_s3_bucket_capabilities(bucket, s3_client_conf.endpoint) + .is_directory_bucket()) { + return Status::NotSupported( + "Presigned URL is not supported for AWS Directory Bucket"); + } + auto s3_client = S3ClientFactory::instance().create(s3_client_conf); + if (s3_client == nullptr) { + return Status::InternalError("Failed to create S3 client for EMBED file input"); + } media_url = s3_client->generate_presigned_url({.bucket = bucket, .key = key}, ttl_seconds, s3_client_conf); return Status::OK(); diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index fa239ca3282e2a..083d0e905b2b29 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -50,6 +50,7 @@ struct ObjectStoragePathOptions { struct ObjectCompleteMultiPart { int part_num = 0; std::string etag = std::string(); + std::optional checksum_crc32c = std::nullopt; }; struct ObjectStorageStatus { @@ -76,6 +77,7 @@ struct ObjectStorageUploadResponse { ObjectStorageResponse resp {}; std::optional upload_id = std::nullopt; std::optional etag = std::nullopt; + std::optional checksum_crc32c = std::nullopt; }; struct ObjectStorageHeadResponse : ObjectStorageResponse { @@ -106,6 +108,11 @@ class ObjStorageClient { virtual ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) = 0; + // Abort an unfinished multipart upload. Backends without an explicit abort primitive may + // retain their existing no-op cleanup semantics. + virtual ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions&) { + return ObjectStorageResponse::OK(); + } // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. // If it exists, it will return the corresponding file size virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index f8b836607a14a6..7f4dd80c9f39c4 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -85,7 +85,12 @@ S3FileWriter::~S3FileWriter() { _wait_until_finish(fmt::format("wait s3 file {} upload to be finished", _obj_storage_path_opts.path.native())); } - // We won't do S3 abort operation in BE, we let s3 service do it own. + if (_obj_storage_path_opts.upload_id.has_value() && state() == State::OPENED) { + s3_bvar::s3_multipart_unfinished_on_destroy_total << 1; + LOG(WARNING) << "S3FileWriter destroyed with an unfinished multipart upload, bucket=" + << _obj_storage_path_opts.bucket << ", key=" << _obj_storage_path_opts.key + << ", upload_id=" << *_obj_storage_path_opts.upload_id; + } if (state() == State::OPENED && !_failed) { s3_bytes_written_total << _bytes_appended; } @@ -283,40 +288,67 @@ Status S3FileWriter::_submit_upload_buffer(const std::shared_ptr& bu Status S3FileWriter::_close_impl() { VLOG_DEBUG << "S3FileWriter::close, path: " << _obj_storage_path_opts.path.native(); - DBUG_EXECUTE_IF("S3FileWriter._close_impl.inject_error", { - if (_obj_storage_path_opts.key.ends_with(".dat")) { - return Status::IOError("S3FileWriter._close_impl.inject_error"); + auto close_status = [&]() -> Status { + DBUG_EXECUTE_IF("S3FileWriter._close_impl.inject_error", { + if (_obj_storage_path_opts.key.ends_with(".dat")) { + return Status::IOError("S3FileWriter._close_impl.inject_error"); + } + }); + + if (_cur_part_num == 1 && + _pending_buf) { // data size is less than config::s3_write_buffer_size + RETURN_IF_ERROR(_set_upload_to_remote_less_than_buffer_size()); } - }); - if (_cur_part_num == 1 && _pending_buf) { // data size is less than config::s3_write_buffer_size - RETURN_IF_ERROR(_set_upload_to_remote_less_than_buffer_size()); - } + if (_bytes_appended == 0) { + DCHECK_EQ(_cur_part_num, 1); + // No data written, but need to create an empty file + RETURN_IF_ERROR(_build_upload_buffer()); + if (!_used_by_s3_committer) { + auto* pending_buf = dynamic_cast(_pending_buf.get()); + pending_buf->set_upload_to_remote( + [this](UploadFileBuffer& buf) { _put_object(buf); }); + } else { + RETURN_IF_ERROR(_create_multi_upload_request()); + } + } - if (_bytes_appended == 0) { - DCHECK_EQ(_cur_part_num, 1); - // No data written, but need to create an empty file - RETURN_IF_ERROR(_build_upload_buffer()); - if (!_used_by_s3_committer) { - auto* pending_buf = dynamic_cast(_pending_buf.get()); - pending_buf->set_upload_to_remote([this](UploadFileBuffer& buf) { _put_object(buf); }); - } else { - RETURN_IF_ERROR(_create_multi_upload_request()); + if (_pending_buf != nullptr) { // there is remaining data in buffer need to be uploaded + auto st = _submit_upload_buffer(_pending_buf); + _pending_buf = nullptr; + if (!st.ok()) { + _wait_until_finish("pending buffer submit failed"); + return st; + } } - } - if (_pending_buf != nullptr) { // there is remaining data in buffer need to be uploaded - auto st = _submit_upload_buffer(_pending_buf); - _pending_buf = nullptr; - if (!st.ok()) { - _wait_until_finish("pending buffer submit failed"); - return st; + RETURN_IF_ERROR(_complete()); + SYNC_POINT_RETURN_WITH_VALUE("s3_file_writer::close", Status()); + return Status::OK(); + }(); + + if (!close_status.ok() && _obj_storage_path_opts.upload_id.has_value()) { + _wait_until_finish("abort after close failure"); + auto abort_status = _abort(); + if (!abort_status.ok()) { + close_status.append(fmt::format("; AbortMultipartUpload also failed: {}", + abort_status.to_string_no_stack())); } } + return close_status; +} - RETURN_IF_ERROR(_complete()); - SYNC_POINT_RETURN_WITH_VALUE("s3_file_writer::close", Status()); - +Status S3FileWriter::_abort() { + DCHECK(_obj_storage_path_opts.upload_id.has_value()); + const auto& client = _obj_client->get(); + if (client == nullptr) { + return Status::InternalError("invalid obj storage client while aborting MPU"); + } + auto resp = client->abort_multipart_upload(_obj_storage_path_opts); + if (resp.status.code != ErrorCode::OK) { + return {resp.status.code, std::move(resp.status.msg)}; + } + _obj_storage_path_opts.upload_id.reset(); return Status::OK(); } @@ -398,7 +430,8 @@ void S3FileWriter::_upload_one_part(int part_num, UploadFileBuffer& buf) { s3_bytes_written_total << buf.get_size(); ObjectCompleteMultiPart completed_part { - part_num, resp.etag.has_value() ? std::move(resp.etag.value()) : ""}; + part_num, resp.etag.has_value() ? std::move(resp.etag.value()) : "", + std::move(resp.checksum_crc32c)}; std::unique_lock lck {_completed_lock}; _completed_parts.emplace_back(std::move(completed_part)); @@ -471,11 +504,6 @@ Status S3FileWriter::_complete() { _wait_until_finish("Complete"); TEST_SYNC_POINT_CALLBACK("S3FileWriter::_complete:1", std::make_pair(&_failed, &_completed_parts)); - if (_used_by_s3_committer) { // S3 committer will complete multipart upload file on FE side. - s3_file_created_total << 1; // Assume that it will be created successfully - return Status::OK(); - } - // check number of parts int64_t expected_num_parts1 = (_bytes_appended / config::s3_write_buffer_size) + !!(_bytes_appended % config::s3_write_buffer_size); @@ -498,6 +526,18 @@ Status S3FileWriter::_complete() { // make sure _completed_parts are ascending order std::sort(_completed_parts.begin(), _completed_parts.end(), [](auto& p1, auto& p2) { return p1.part_num < p2.part_num; }); + for (size_t i = 0; i < _completed_parts.size(); ++i) { + if (_completed_parts[i].part_num != static_cast(i + 1)) { + return Status::InternalError( + "multipart upload parts must be consecutive from 1, completed_parts_list={} " + "file_path={}", + _dump_completed_part(), _obj_storage_path_opts.path.native()); + } + } + if (_used_by_s3_committer) { // S3 committer will complete multipart upload file on FE side. + s3_file_created_total << 1; // Assume that it will be created successfully + return Status::OK(); + } TEST_SYNC_POINT_CALLBACK("S3FileWriter::_complete:2", &_completed_parts); LOG(INFO) << "complete_multipart_upload " << _obj_storage_path_opts.path.native() << " size=" << _bytes_appended << " number_parts=" << _completed_parts.size() diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 1d1f146cd6244f..ebed68001a701a 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -128,18 +128,38 @@ Aws::String compute_crc32c_b64(std::string_view data) { static_cast((crc >> 8) & 0xff), static_cast(crc & 0xff)}; return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::ByteBuffer(bytes, sizeof(bytes))); } + +std::string directory_list_prefix(std::string_view prefix) { + if (prefix.empty() || prefix.ends_with('/')) { + return std::string(prefix); + } + const auto slash = prefix.rfind('/'); + return slash == std::string_view::npos ? "" : std::string(prefix.substr(0, slash + 1)); +} + +std::string delete_errors_message(const Aws::Vector& errors, + const Aws::String& request_id) { + std::string message = fmt::format("DeleteObjects partially failed, request_id={}", request_id); + for (const auto& error : errors) { + fmt::format_to(std::back_inserter(message), ", key={}, code={}, message={}", error.GetKey(), + error.GetCode(), error.GetMessage()); + } + return message; +} } // namespace S3ObjStorageClient::S3ObjStorageClient(std::shared_ptr client, - bool is_s3_express) - : _client(std::move(client)), - _disable_content_md5(config::s3_disable_content_md5 || is_s3_express) {} + S3BucketCapabilities capabilities) + : _client(std::move(client)), _capabilities(capabilities) {} ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { CreateMultipartUploadRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); request.SetContentType("application/octet-stream"); + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { + request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); + } MonotonicStopWatch watch; watch.start(); @@ -177,7 +197,7 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti Aws::S3::Model::PutObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); auto string_view_stream = std::make_shared(stream.data(), stream.size()); - if (_disable_content_md5) { + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { // S3 Express One Zone rejects Content-MD5; use CRC32C instead. request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); request.SetChecksumCRC32C(compute_crc32c_b64(stream)); @@ -228,10 +248,12 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP request.SetBody(string_view_stream); - if (_disable_content_md5) { + std::optional checksum_crc32c; + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { // S3 Express One Zone rejects Content-MD5; use CRC32C instead. + checksum_crc32c = compute_crc32c_b64(stream); request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); - request.SetChecksumCRC32C(compute_crc32c_b64(stream)); + request.SetChecksumCRC32C(*checksum_crc32c); } else { Aws::Utils::ByteBuffer part_md5( Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); @@ -273,7 +295,23 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP << "UploadPart cost=" << watch.elapsed_time_milliseconds() << "ms" << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key << ", part_num=" << part_num << ", upload_id=" << *opts.upload_id; - return ObjectStorageUploadResponse {.etag = outcome.GetResult().GetETag()}; + if (checksum_crc32c.has_value() && + outcome.GetResult().GetChecksumCRC32C() != *checksum_crc32c) { + s3_bvar::s3_checksum_failure_total << 1; + auto st = Status::IOError( + "CRC32C mismatch for UploadPart, bucket={}, key={}, part={}, expected={}, " + "actual={}, request_id={}", + opts.bucket, opts.key, part_num, *checksum_crc32c, + outcome.GetResult().GetChecksumCRC32C(), request_id); + return ObjectStorageUploadResponse { + .resp = {convert_to_obj_response(std::move(st)), 200, request_id}}; + } + return ObjectStorageUploadResponse { + .etag = outcome.GetResult().GetETag(), + .checksum_crc32c = checksum_crc32c.has_value() + ? std::optional(std::string( + checksum_crc32c->c_str(), checksum_crc32c->size())) + : std::nullopt}; } ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( @@ -282,13 +320,31 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( CompleteMultipartUploadRequest request; request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { + for (size_t i = 0; i < completed_parts.size(); ++i) { + const auto& part = completed_parts[i]; + if (part.part_num != static_cast(i + 1)) { + return {convert_to_obj_response(Status::InvalidArgument( + "Directory Bucket multipart parts must be consecutive from 1"))}; + } + if (!part.checksum_crc32c.has_value()) { + return {convert_to_obj_response(Status::InvalidArgument( + "AWS Directory Bucket multipart completion requires CRC32C for part {}", + part.part_num))}; + } + } + } + CompletedMultipartUpload completed_upload; std::vector complete_parts; std::ranges::transform(completed_parts, std::back_inserter(complete_parts), - [](const ObjectCompleteMultiPart& part_ptr) { + [this](const ObjectCompleteMultiPart& part_ptr) { CompletedPart part; part.SetPartNumber(part_ptr.part_num); part.SetETag(part_ptr.etag); + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { + part.SetChecksumCRC32C(*part_ptr.checksum_crc32c); + } return part; }); completed_upload.SetParts(std::move(complete_parts)); @@ -324,6 +380,25 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( return ObjectStorageResponse::OK(); } +ObjectStorageResponse S3ObjStorageClient::abort_multipart_upload( + const ObjectStoragePathOptions& opts) { + Aws::S3::Model::AbortMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); + auto outcome = s3_put_rate_limit([&]() { return _client->AbortMultipartUpload(request); }); + s3_bvar::s3_multipart_abort_total << 1; + if (outcome.IsSuccess() || + outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return ObjectStorageResponse::OK(); + } + s3_bvar::s3_multipart_abort_failure_total << 1; + return {convert_to_obj_response(s3fs_error( + outcome.GetError(), + fmt::format("failed to AbortMultipartUpload: {}, upload_id={}", + opts.path.native(), *opts.upload_id))), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetRequestId()}; +} + ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { Aws::S3::Model::HeadObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); @@ -376,7 +451,10 @@ ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOpti ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, std::vector* files) { Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket).WithPrefix(opts.prefix); + std::string service_prefix = _capabilities.is_directory_bucket() + ? directory_list_prefix(opts.prefix) + : opts.prefix; + request.WithBucket(opts.bucket).WithPrefix(service_prefix); bool is_trucated = false; do { Aws::S3::Model::ListObjectsV2Outcome outcome; @@ -400,14 +478,25 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp static_cast(outcome.GetError().GetResponseCode()), outcome.GetError().GetRequestId()}; } + if (_capabilities.is_directory_bucket()) { + s3_bvar::s3_directory_list_pages << 1; + s3_bvar::s3_directory_list_scanned_keys + << outcome.GetResult().GetContents().size(); + } for (const auto& obj : outcome.GetResult().GetContents()) { std::string key = obj.GetKey(); + if (!key.starts_with(opts.prefix)) { + continue; + } bool is_dir = (key.back() == '/'); FileInfo file_info; file_info.file_name = obj.GetKey(); file_info.file_size = obj.GetSize(); file_info.is_file = !is_dir; files->push_back(std::move(file_info)); + if (_capabilities.is_directory_bucket()) { + s3_bvar::s3_directory_list_returned_keys << 1; + } } is_trucated = outcome.GetResult().GetIsTruncated(); if (is_trucated && outcome.GetResult().GetNextContinuationToken().empty()) { @@ -419,6 +508,9 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); } while (is_trucated); + if (!_capabilities.list_is_lexicographic) { + std::ranges::sort(*files, {}, &FileInfo::file_name); + } return ObjectStorageResponse::OK(); } @@ -426,6 +518,9 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath std::vector objs) { Aws::S3::Model::DeleteObjectsRequest delete_request; delete_request.SetBucket(opts.bucket); + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { + delete_request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); + } Aws::S3::Model::Delete del; Aws::Vector objects; std::ranges::transform(objs, std::back_inserter(objects), [](auto&& obj_key) { @@ -448,10 +543,9 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath // case for partial delete object failure SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); if (!delete_outcome.GetResult().GetErrors().empty()) { - const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response( - Status::InternalError("failed to delete object {}: {}, request_id={}", e.GetKey(), - e.GetMessage(), delete_outcome.GetResult().GetRequestId()))}; + return {convert_to_obj_response(Status::InternalError(delete_errors_message( + delete_outcome.GetResult().GetErrors(), + delete_outcome.GetResult().GetRequestId())))}; } return ObjectStorageResponse::OK(); } @@ -475,9 +569,15 @@ ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathO ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( const ObjectStoragePathOptions& opts) { Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket).WithPrefix(opts.prefix); + request.WithBucket(opts.bucket) + .WithPrefix(_capabilities.is_directory_bucket() + ? directory_list_prefix(opts.prefix) + : opts.prefix); Aws::S3::Model::DeleteObjectsRequest delete_request; delete_request.SetBucket(opts.bucket); + if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { + delete_request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); + } bool is_trucated = false; do { Aws::S3::Model::ListObjectsV2Outcome outcome; @@ -496,8 +596,11 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( Aws::Vector objects; objects.reserve(result.GetContents().size()); for (const auto& obj : result.GetContents()) { - objects.emplace_back().SetKey(obj.GetKey()); + if (obj.GetKey().starts_with(opts.prefix)) { + objects.emplace_back().SetKey(obj.GetKey()); + } } + bool deleted_objects = false; if (!objects.empty()) { Aws::S3::Model::Delete del; del.WithObjects(std::move(objects)).SetQuiet(true); @@ -516,14 +619,16 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", &delete_outcome); if (!delete_outcome.GetResult().GetErrors().empty()) { - const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response(Status::InternalError( - "failed to delete object {}: {}, request_id={}", opts.key, e.GetMessage(), - delete_outcome.GetResult().GetRequestId()))}; + return {convert_to_obj_response(Status::InternalError(delete_errors_message( + delete_outcome.GetResult().GetErrors(), + delete_outcome.GetResult().GetRequestId())))}; } + deleted_objects = true; } is_trucated = result.GetIsTruncated(); - request.SetContinuationToken(result.GetNextContinuationToken()); + request.SetContinuationToken(_capabilities.is_directory_bucket() && deleted_objects + ? Aws::String() + : result.GetNextContinuationToken()); } while (is_trucated); return ObjectStorageResponse::OK(); } @@ -531,6 +636,11 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, int64_t expiration_secs, const S3ClientConf&) { + if (!_capabilities.supports_presign) { + LOG(WARNING) << "Presigned URL is not supported for AWS Directory Bucket, bucket=" + << opts.bucket; + return {}; + } return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, expiration_secs); } diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 0216318936e4f4..27ce8686dd1e5c 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -17,6 +17,7 @@ #pragma once +#include "cpp/s3_bucket_capabilities.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_file_system.h" @@ -33,7 +34,7 @@ class ObjClientHolder; class S3ObjStorageClient final : public ObjStorageClient { public: explicit S3ObjStorageClient(std::shared_ptr client, - bool is_s3_express = false); + S3BucketCapabilities capabilities = {}); ~S3ObjStorageClient() override = default; ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override; @@ -44,6 +45,7 @@ class S3ObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; + ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, @@ -59,10 +61,7 @@ class S3ObjStorageClient final : public ObjStorageClient { private: std::shared_ptr _client; - // True for S3 Express One Zone endpoints (or when config::s3_disable_content_md5 - // is on). When set, uploads send a CRC32C checksum instead of Content-MD5, - // since S3 Express returns 501 NotImplemented for the latter. - bool _disable_content_md5 = false; + S3BucketCapabilities _capabilities; }; } // namespace doris::io diff --git a/be/src/io/tools/file_cache_microbench.cpp b/be/src/io/tools/file_cache_microbench.cpp index 46dfb6934fd928..ccc8e60f3899b2 100644 --- a/be/src/io/tools/file_cache_microbench.cpp +++ b/be/src/io/tools/file_cache_microbench.cpp @@ -1495,6 +1495,7 @@ class JobManager { s3_conf.sk = doris::config::test_s3_sk; s3_conf.region = doris::config::test_s3_region; s3_conf.endpoint = doris::config::test_s3_endpoint; + s3_conf.bucket = doris::config::test_s3_bucket; return s3_conf; } diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index 028613f5533dc4..af869772f94766 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -78,18 +78,64 @@ bvar::LatencyRecorder s3_list_latency("s3_list"); bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); +bvar::Adder s3_checksum_failure_total("s3_checksum_failure_total"); +bvar::Adder s3_multipart_abort_total("s3_multipart_abort_total"); +bvar::Adder s3_multipart_abort_failure_total("s3_multipart_abort_failure_total"); +bvar::Adder s3_multipart_unfinished_on_destroy_total( + "s3_multipart_unfinished_on_destroy_total"); +bvar::Adder s3_directory_list_scanned_keys("s3_directory_list_scanned_keys"); +bvar::Adder s3_directory_list_returned_keys("s3_directory_list_returned_keys"); +bvar::Adder s3_directory_list_pages("s3_directory_list_pages"); }; // namespace s3_bvar namespace { doris::Status is_s3_conf_valid(const S3ClientConf& conf) { - if (conf.endpoint.empty()) { + if (conf.endpoint.empty() && conf.provider == io::ObjStorageType::AZURE) { return Status::InvalidArgument("Invalid s3 conf, empty endpoint"); } if (conf.region.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } + const auto capabilities = resolve_s3_bucket_capabilities(conf.bucket, conf.endpoint); + if (capabilities.official_aws_service && is_s3express_control_endpoint(conf.endpoint)) { + return Status::InvalidArgument( + "s3express-control endpoint cannot be used for object operations"); + } + if (capabilities.official_aws_service && is_s3express_zonal_endpoint(conf.endpoint) && + !capabilities.is_directory_bucket()) { + return Status::InvalidArgument( + "An S3 Express endpoint requires a valid AWS Directory Bucket name"); + } + if (capabilities.is_directory_bucket()) { + if (capabilities.endpoint_mode != S3EndpointMode::AWS_SDK_RULES) { + return Status::InvalidArgument( + "AWS Directory Bucket requires a standard regional or zonal S3 endpoint"); + } + if (!conf.use_virtual_addressing) { + return Status::InvalidArgument( + "Path-style addressing is not supported for AWS Directory Bucket"); + } + if (config::s3_client_http_scheme == "http" || conf.endpoint.starts_with("http://")) { + return Status::InvalidArgument("AWS Directory Bucket requires HTTPS"); + } + const auto endpoint_region = aws_s3_endpoint_region(conf.endpoint); + if (!endpoint_region.empty() && endpoint_region != conf.region) { + return Status::InvalidArgument( + "Configured endpoint region {} does not match AWS_REGION {} for Directory " + "Bucket {}", + endpoint_region, conf.region, conf.bucket); + } + const auto endpoint_zone = s3express_endpoint_zone_id(conf.endpoint); + const auto bucket_zone = s3_directory_bucket_zone_id(conf.bucket); + if (!endpoint_zone.empty() && endpoint_zone != bucket_zone) { + return Status::InvalidArgument( + "Configured endpoint zone {} does not match Directory Bucket zone {}", + endpoint_zone, bucket_zone); + } + } + if (conf.role_arn.empty()) { // Allow anonymous access when both ak and sk are empty bool hasAk = !conf.ak.empty(); @@ -495,7 +541,15 @@ std::shared_ptr S3ClientFactory::_create_s3_client( // endpoint-rules resolver is active. This is required for S3 Express One Zone: // the resolver detects the --x-s3 bucket suffix, calls CreateSession, and signs // requests with service name "s3express" instead of "s3". - Aws::S3::S3ClientConfiguration aws_config; + const auto capabilities = + resolve_s3_bucket_capabilities(s3_conf.bucket, s3_conf.endpoint); + const auto payload_signing_policy = + capabilities.is_directory_bucket() + ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent + : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; + Aws::S3::S3ClientConfiguration aws_config(S3ClientFactory::getClientConfiguration(), + payload_signing_policy, + s3_conf.use_virtual_addressing); aws_config.region = s3_conf.region; aws_config.useVirtualAddressing = s3_conf.use_virtual_addressing; @@ -513,7 +567,6 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.maxConnections = 102400; } - aws_config.requestTimeoutMs = 30000; if (s3_conf.request_timeout_ms > 0) { aws_config.requestTimeoutMs = s3_conf.request_timeout_ms; } @@ -529,21 +582,26 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.retryStrategy = std::make_shared( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/false); - // S3 Express buckets are identified by the --x-s3 suffix or an s3express endpoint. - // Skip endpointOverride for these so the SDK resolves the bucket-specific endpoint - // and manages CreateSession automatically. For all other buckets, keep the override. - const bool express = is_s3_express(s3_conf.bucket, s3_conf.endpoint); - if (s3_conf.need_override_endpoint && !express) { + // Directory Bucket requests must remain on the SDK endpoint-rules path so it can resolve + // the bucket-specific Zonal host and manage CreateSession credentials. + if (s3_conf.need_override_endpoint && !s3_conf.endpoint.empty() && + !capabilities.is_directory_bucket()) { aws_config.endpointOverride = s3_conf.endpoint; } - aws_config.disableS3ExpressAuth = !express; + aws_config.disableS3ExpressAuth = !capabilities.official_aws_service; auto new_client = std::make_shared( get_aws_credentials_provider(s3_conf), Aws::MakeShared("S3Client"), aws_config); - auto obj_client = std::make_shared(std::move(new_client), express); - LOG_INFO("create one s3 client with {}", s3_conf.to_string()); + auto obj_client = + std::make_shared(std::move(new_client), capabilities); + LOG_INFO("create one s3 client with {}, bucket_type={}, endpoint_mode={}, checksum_policy={}", + s3_conf.to_string(), + capabilities.is_directory_bucket() ? "directory" : "general_purpose", + capabilities.endpoint_mode == S3EndpointMode::AWS_SDK_RULES ? "sdk_rules" + : "override", + capabilities.checksum_policy == S3ChecksumPolicy::CRC32C ? "crc32c" : "content_md5"); return obj_client; } @@ -805,18 +863,7 @@ std::string hide_access_key(const std::string& ak) { } bool is_s3_express(std::string_view bucket, std::string_view endpoint) { - constexpr std::string_view bucket_suffix = "--x-s3"; - if (bucket.ends_with(bucket_suffix)) { - return true; - } - - const auto scheme_pos = endpoint.find("://"); - if (scheme_pos != std::string_view::npos) { - endpoint.remove_prefix(scheme_pos + 3); - } - endpoint = endpoint.substr(0, endpoint.find('/')); - return endpoint.starts_with("s3express-") || - endpoint.find(".s3express-") != std::string_view::npos; + return resolve_s3_bucket_capabilities(bucket, endpoint).is_directory_bucket(); } } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 950600fe8a534f..5a849b61734e3f 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -35,6 +35,7 @@ #include "common/status.h" #include "core/string_ref.h" #include "cpp/aws_common.h" +#include "cpp/s3_bucket_capabilities.h" #include "cpp/token_bucket_rate_limiter.h" #include "io/fs/obj_storage_client.h" @@ -60,6 +61,13 @@ extern bvar::LatencyRecorder s3_list_latency; extern bvar::LatencyRecorder s3_list_object_versions_latency; extern bvar::LatencyRecorder s3_get_bucket_version_latency; extern bvar::LatencyRecorder s3_copy_object_latency; +extern bvar::Adder s3_checksum_failure_total; +extern bvar::Adder s3_multipart_abort_total; +extern bvar::Adder s3_multipart_abort_failure_total; +extern bvar::Adder s3_multipart_unfinished_on_destroy_total; +extern bvar::Adder s3_directory_list_scanned_keys; +extern bvar::Adder s3_directory_list_returned_keys; +extern bvar::Adder s3_directory_list_pages; }; // namespace s3_bvar std::string hide_access_key(const std::string& ak); @@ -92,21 +100,29 @@ struct S3ClientConf { uint64_t get_hash() const { uint64_t hash_code = 0; - // Use crc32_hash(ak + sk) hash to prevent swapped AK/SK order from producing same result. - hash_code ^= crc32_hash(ak + sk); - hash_code ^= crc32_hash(token); - hash_code ^= crc32_hash(endpoint); - hash_code ^= crc32_hash(region); - hash_code ^= crc32_hash(bucket); - hash_code ^= max_connections; - hash_code ^= request_timeout_ms; - hash_code ^= connect_timeout_ms; - hash_code ^= use_virtual_addressing; - hash_code ^= static_cast(provider); - - hash_code ^= static_cast(cred_provider_type); - hash_code ^= crc32_hash(role_arn); - hash_code ^= crc32_hash(external_id); + auto combine = [&hash_code](uint64_t value) { + hash_code ^= value + 0x9e3779b97f4a7c15ULL + (hash_code << 6) + (hash_code >> 2); + }; + combine(crc32_hash(ak)); + combine(crc32_hash(sk)); + combine(crc32_hash(token)); + combine(crc32_hash(endpoint)); + combine(crc32_hash(region)); + combine(crc32_hash(bucket)); + combine(max_connections); + combine(request_timeout_ms); + combine(connect_timeout_ms); + combine(use_virtual_addressing); + combine(need_override_endpoint); + combine(static_cast(provider)); + combine(static_cast(cred_provider_type)); + combine(crc32_hash(role_arn)); + combine(crc32_hash(external_id)); + const auto capabilities = resolve_s3_bucket_capabilities(bucket, endpoint); + combine(static_cast(capabilities.bucket_type)); + combine(static_cast(capabilities.endpoint_mode)); + combine(static_cast(capabilities.checksum_policy)); + combine(crc32_hash(config::s3_client_http_scheme)); return hash_code; } @@ -114,10 +130,12 @@ struct S3ClientConf { return fmt::format( "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " - "cred_provider_type={},role_arn={}, external_id={}", - hide_access_key(ak), token, endpoint, region, bucket, max_connections, - request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, - role_arn, external_id); + "need_override_endpoint={}, cred_provider_type={}, role_arn={}, external_id={})", + hide_access_key(ak), token.empty() ? "" : "***", endpoint, region, bucket, + max_connections, + request_timeout_ms, connect_timeout_ms, use_virtual_addressing, + need_override_endpoint, cred_provider_type, role_arn, + external_id.empty() ? "" : "***"); } }; diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 13f9d12c501197..0d54dbdda86b01 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -1140,6 +1140,9 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { const std::vector& completed_parts) override { std::lock_guard lock(_mutex); complete_multipart_count++; + if (fail_complete_multipart) { + return {.status = {.code = 1, .msg = "injected complete multipart failure"}}; + } complete_multipart_params.push_back({opts, completed_parts}); last_opts = opts; last_completed_parts = completed_parts; @@ -1153,6 +1156,14 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } + ObjectStorageResponse abort_multipart_upload( + const ObjectStoragePathOptions& opts) override { + std::lock_guard lock(_mutex); + abort_multipart_count++; + last_opts = opts; + return default_response; + } + ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { std::lock_guard lock(_mutex); return {.resp = ObjectStorageResponse::OK(), @@ -1227,6 +1238,8 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { int put_object_count = 0; int upload_part_count = 0; int complete_multipart_count = 0; + int abort_multipart_count = 0; + bool fail_complete_multipart = false; // Structures to store input parameters for each call struct UploadPartParams { @@ -1265,6 +1278,8 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { put_object_count = 0; upload_part_count = 0; complete_multipart_count = 0; + abort_multipart_count = 0; + fail_complete_multipart = false; create_multipart_params.clear(); put_object_params.clear(); @@ -1306,6 +1321,20 @@ create_s3_client(const std::string& path) { return {mock_client, s3_file_writer}; } +TEST_F(S3FileWriterTest, abortMultipartUploadWhenCompleteFails) { + auto [client, writer] = create_s3_client("abort_on_complete_failure"); + std::string content(config::s3_write_buffer_size + 1, 'x'); + ASSERT_TRUE(writer->append(Slice(content)).ok()); + client->fail_complete_multipart = true; + + auto status = writer->close(); + + EXPECT_FALSE(status.ok()); + EXPECT_EQ(1, client->complete_multipart_count); + EXPECT_EQ(1, client->abort_multipart_count); + EXPECT_TRUE(writer->upload_id().empty()); +} + /** * Generate test data for S3FileWriter boundary tests. * Returns a vector of sizes that we'll use to generate data on demand. diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index cba03bec576773..cb9da739899557 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -78,11 +78,45 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { } TEST_F(S3UTILTest, is_s3_express_context) { - EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "")); - EXPECT_TRUE(is_s3_express("", "bucket.s3express-use1-az4.us-east-1.amazonaws.com")); - EXPECT_FALSE(is_s3_express("bucket--x-s3-suffix", "")); + EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", + "https://s3.us-east-1.amazonaws.com")); + EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", + "bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com")); + EXPECT_FALSE(is_s3_express("bucket--x-s3-suffix", "s3.us-east-1.amazonaws.com")); EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path")); - EXPECT_FALSE(is_s3_express("bucket", "s3.us-east-1.amazonaws.com")); +} + +TEST_F(S3UTILTest, s3_directory_bucket_capabilities) { + auto capabilities = resolve_s3_bucket_capabilities( + "analytics--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com"); + EXPECT_TRUE(capabilities.is_directory_bucket()); + EXPECT_TRUE(capabilities.official_aws_service); + EXPECT_EQ(S3EndpointMode::AWS_SDK_RULES, capabilities.endpoint_mode); + EXPECT_EQ(S3ChecksumPolicy::CRC32C, capabilities.checksum_policy); + EXPECT_FALSE(capabilities.supports_start_after); + EXPECT_FALSE(capabilities.list_is_lexicographic); + EXPECT_FALSE(capabilities.supports_versioning); + EXPECT_FALSE(capabilities.supports_presign); + + capabilities = resolve_s3_bucket_capabilities("analytics--use1-az4--x-s3", ""); + EXPECT_TRUE(capabilities.is_directory_bucket()); + EXPECT_EQ("use1-az4", s3_directory_bucket_zone_id("analytics--use1-az4--x-s3")); + EXPECT_EQ("use1-az4", s3express_endpoint_zone_id( + "analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1." + "amazonaws.com")); + EXPECT_EQ("us-east-1", aws_s3_endpoint_region("s3.us-east-1.amazonaws.com")); + + capabilities = resolve_s3_bucket_capabilities( + "analytics--use1-az4--x-s3", "https://s3.dualstack.us-east-1.amazonaws.com"); + EXPECT_TRUE(capabilities.is_directory_bucket()); + EXPECT_EQ(S3EndpointMode::EXPLICIT_OVERRIDE, capabilities.endpoint_mode); + + capabilities = resolve_s3_bucket_capabilities("analytics--use1-az4--x-s3", + "https://minio.example.com"); + EXPECT_FALSE(capabilities.is_directory_bucket()); + EXPECT_FALSE(capabilities.official_aws_service); + EXPECT_EQ(S3ChecksumPolicy::CONTENT_MD5, capabilities.checksum_policy); } // Verifies that check_s3_rate_limiter_config_changed() rebuilds the global GET rate diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 2e19ebbb8a3bff..179ab9679b05d2 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -38,6 +38,7 @@ #include "common/network_util.h" #include "common/stats.h" #include "common/string_util.h" +#include "cpp/s3_bucket_capabilities.h" #include "cpp/sync_point.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" @@ -768,6 +769,21 @@ static bool use_credential_provider(const ObjectStoreInfoPB& obj) { return obj.has_cred_provider_type() || has_non_empty_role_arn(obj); } +static bool is_unsupported_directory_bucket_vault(const ObjectStoreInfoPB& obj) { + return obj.has_provider() && obj.provider() == ObjectStoreInfoPB::S3 && + resolve_s3_bucket_capabilities(obj.bucket(), obj.endpoint()).is_directory_bucket(); +} + +static int reject_directory_bucket_vault(const ObjectStoreInfoPB& obj, MetaServiceCode& code, + std::string& msg) { + if (!is_unsupported_directory_bucket_vault(obj)) { + return 0; + } + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "S3 Express One Zone is not supported by Cloud Storage Vault in this release"; + return -1; +} + static void create_object_info_with_encrypt(const InstanceInfoPB& instance, ObjectStoreInfoPB* obj, bool sse_enabled, MetaServiceCode& code, std::string& msg) { @@ -781,6 +797,10 @@ static void create_object_info_with_encrypt(const InstanceInfoPB& instance, Obje std::string external_endpoint = obj->has_external_endpoint() ? obj->external_endpoint() : ""; std::string region = obj->has_region() ? obj->region() : ""; + if (reject_directory_bucket_vault(*obj, code, msg) != 0) { + return; + } + if (obj->has_role_arn()) { if (obj->role_arn().empty() || !obj->has_cred_provider_type() || !obj->has_provider() || obj->provider() != ObjectStoreInfoPB::S3 || bucket.empty() || endpoint.empty() || @@ -1105,6 +1125,9 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt msg = ss.str(); return -1; } + if (reject_directory_bucket_vault(new_vault.obj_info(), code, msg) != 0) { + return -1; + } auto origin_vault_info = new_vault.DebugString(); @@ -1248,6 +1271,10 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, const auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info(); + if (reject_directory_bucket_vault(obj, code, msg) != 0) { + return -1; + } + // obj size > 1k, refuse if (obj.ByteSizeLong() > 1024) { code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/common/cpp/s3_bucket_capabilities.h b/common/cpp/s3_bucket_capabilities.h new file mode 100644 index 00000000000000..999354a195895a --- /dev/null +++ b/common/cpp/s3_bucket_capabilities.h @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace doris { + +enum class S3BucketType { GENERAL_PURPOSE, DIRECTORY }; +enum class S3EndpointMode { EXPLICIT_OVERRIDE, AWS_SDK_RULES }; +enum class S3ChecksumPolicy { CONTENT_MD5, CRC32C }; + +struct S3BucketCapabilities { + S3BucketType bucket_type = S3BucketType::GENERAL_PURPOSE; + S3EndpointMode endpoint_mode = S3EndpointMode::EXPLICIT_OVERRIDE; + S3ChecksumPolicy checksum_policy = S3ChecksumPolicy::CONTENT_MD5; + bool official_aws_service = false; + bool require_https = false; + bool require_virtual_addressing = false; + bool supports_start_after = true; + bool list_is_lexicographic = true; + bool supports_versioning = true; + bool supports_presign = true; + + bool is_directory_bucket() const { return bucket_type == S3BucketType::DIRECTORY; } +}; + +inline std::string s3_endpoint_host(std::string_view endpoint) { + const auto scheme = endpoint.find("://"); + if (scheme != std::string_view::npos) { + endpoint.remove_prefix(scheme + 3); + } + endpoint = endpoint.substr(0, endpoint.find_first_of("/?#")); + if (const auto port = endpoint.find(':'); port != std::string_view::npos) { + endpoint = endpoint.substr(0, port); + } + std::string host(endpoint); + std::transform(host.begin(), host.end(), host.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + while (!host.empty() && host.back() == '.') { + host.pop_back(); + } + return host; +} + +inline bool is_aws_s3_endpoint(std::string_view endpoint) { + if (endpoint.empty()) { + // An omitted endpoint means that the AWS SDK resolves the public AWS S3 endpoint + // from the configured region. + return true; + } + const std::string host = s3_endpoint_host(endpoint); + if (host.empty()) { + return false; + } + const bool aws_dns = host.ends_with(".amazonaws.com") || + host.ends_with(".amazonaws.com.cn") || host.ends_with(".api.aws"); + if (!aws_dns) { + return false; + } + return host.starts_with("s3.") || host.starts_with("s3-") || + host.starts_with("s3express-") || host.starts_with("s3express-control.") || + host.find(".s3.") != std::string::npos || + host.find(".s3-") != std::string::npos || + host.find(".s3express-") != std::string::npos; +} + +inline bool is_s3express_control_endpoint(std::string_view endpoint) { + const std::string host = s3_endpoint_host(endpoint); + return host.starts_with("s3express-control.") || + host.find(".s3express-control.") != std::string::npos; +} + +inline bool is_s3express_zonal_endpoint(std::string_view endpoint) { + if (is_s3express_control_endpoint(endpoint)) { + return false; + } + const std::string host = s3_endpoint_host(endpoint); + return host.starts_with("s3express-") || host.find(".s3express-") != std::string::npos; +} + +inline bool is_aws_explicit_s3_endpoint(std::string_view endpoint) { + const std::string host = s3_endpoint_host(endpoint); + return host.find("fips") != std::string::npos || host.find("dualstack") != std::string::npos || + host.find("accelerate") != std::string::npos || + host.find("vpce") != std::string::npos || + is_s3express_control_endpoint(endpoint); +} + +inline std::string s3_directory_bucket_zone_id(std::string_view bucket) { + constexpr std::string_view suffix = "--x-s3"; + if (!bucket.ends_with(suffix)) { + return {}; + } + bucket.remove_suffix(suffix.size()); + const auto separator = bucket.rfind("--"); + return separator == std::string_view::npos ? "" : std::string(bucket.substr(separator + 2)); +} + +inline std::string s3express_endpoint_zone_id(std::string_view endpoint) { + if (is_s3express_control_endpoint(endpoint)) { + return {}; + } + const std::string host = s3_endpoint_host(endpoint); + const auto marker = host.find("s3express-"); + if (marker == std::string::npos) { + return {}; + } + const auto begin = marker + std::string_view("s3express-").size(); + const auto end = host.find('.', begin); + return host.substr(begin, end == std::string::npos ? end : end - begin); +} + +inline std::string s3_endpoint_host_label(std::string_view host, std::size_t begin) { + const auto end = host.find('.', begin); + return std::string(host.substr(begin, end == std::string_view::npos ? end : end - begin)); +} + +inline std::string aws_s3_endpoint_region(std::string_view endpoint) { + const std::string host = s3_endpoint_host(endpoint); + if (host.empty() || host == "s3.amazonaws.com") { + return {}; + } + + const auto control = host.find("s3express-control."); + if (control != std::string::npos) { + const auto begin = control + std::string_view("s3express-control.").size(); + return s3_endpoint_host_label(host, begin); + } + const auto express = host.find("s3express-"); + if (express != std::string::npos) { + const auto dot = host.find('.', express); + return dot == std::string::npos ? "" : s3_endpoint_host_label(host, dot + 1); + } + const auto standard = host.find("s3."); + if (standard != std::string::npos) { + auto begin = standard + std::string_view("s3.").size(); + if (host.substr(begin).starts_with("dualstack.")) { + begin += std::string_view("dualstack.").size(); + } + const auto value = s3_endpoint_host_label(host, begin); + return value == "amazonaws" ? "" : value; + } + if (host.starts_with("s3-")) { + const auto begin = std::string_view("s3-").size(); + return s3_endpoint_host_label(host, begin); + } + return {}; +} + +inline bool is_s3_directory_bucket_name(std::string_view bucket) { + static const std::regex pattern( + "^[a-z0-9]([a-z0-9-]*[a-z0-9])?--[a-z0-9]+-az[0-9]+--x-s3$"); + return bucket.size() >= 3 && bucket.size() <= 63 && + std::regex_match(bucket.begin(), bucket.end(), pattern); +} + +inline S3BucketCapabilities resolve_s3_bucket_capabilities(std::string_view bucket, + std::string_view endpoint) { + S3BucketCapabilities capabilities; + capabilities.official_aws_service = is_aws_s3_endpoint(endpoint); + if (!is_s3_directory_bucket_name(bucket) || !capabilities.official_aws_service) { + return capabilities; + } + + capabilities.bucket_type = S3BucketType::DIRECTORY; + capabilities.endpoint_mode = is_aws_explicit_s3_endpoint(endpoint) + ? S3EndpointMode::EXPLICIT_OVERRIDE + : S3EndpointMode::AWS_SDK_RULES; + capabilities.checksum_policy = S3ChecksumPolicy::CRC32C; + capabilities.require_https = true; + capabilities.require_virtual_addressing = true; + capabilities.supports_start_after = false; + capabilities.list_is_lexicographic = false; + capabilities.supports_versioning = false; + capabilities.supports_presign = false; + return capabilities; +} + +} // namespace doris diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java index ae31eb815d11da..bb4713ffdd55d4 100644 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java @@ -50,6 +50,10 @@ public S3FileReader(String accessKey, String secretKey, String endpoint, String this.region = region; S3Utils.parseURI(uri); this.bucketName = S3Utils.getBucket(); + if (S3Utils.isAwsDirectoryBucket(bucketName, endpoint)) { + throw new IOException( + "S3 Express One Zone is not supported by the Avro Hadoop S3A reader"); + } this.key = S3Utils.getKey(); this.accessKey = accessKey; this.secretKey = secretKey; diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java index 45845af3c0392e..06a9d0d8654f2b 100644 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java @@ -20,8 +20,14 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; +import java.util.regex.Pattern; public class S3Utils { + private static final Pattern DIRECTORY_BUCKET_PATTERN = Pattern.compile( + "^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9]+-az[0-9]+--x-s3$"); private static final String SCHEMA_S3 = "s3"; private static final String SCHEMA_HTTP = "http"; private static final String SCHEMA_HTTPS = "https"; @@ -99,4 +105,28 @@ public static String getKey() { return key; } + public static boolean isAwsDirectoryBucket(String bucketName, String endpoint) { + if (bucketName == null || !DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches()) { + return false; + } + if (StringUtils.isBlank(endpoint)) { + return true; + } + String value = endpoint.contains(SCHEME_DELIM) ? endpoint : "https://" + endpoint; + try { + String host = new URI(value).getHost(); + if (host == null) { + return false; + } + host = host.toLowerCase(Locale.ROOT); + boolean awsDns = host.endsWith(".amazonaws.com") + || host.endsWith(".amazonaws.com.cn") || host.endsWith(".api.aws"); + return awsDns && (host.startsWith("s3.") || host.startsWith("s3-") + || host.startsWith("s3express-") || host.contains(".s3.") + || host.contains(".s3-") || host.contains(".s3express-")); + } catch (URISyntaxException e) { + return false; + } + } + } diff --git a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java b/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java new file mode 100644 index 00000000000000..a331c6447bc4ca --- /dev/null +++ b/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.avro; + +import org.junit.Assert; +import org.junit.Test; + +public class S3UtilsTest { + + @Test + public void testDirectoryBucketClassification() { + String bucket = "analytics--use1-az4--x-s3"; + Assert.assertTrue(S3Utils.isAwsDirectoryBucket( + bucket, "https://s3.us-east-1.amazonaws.com")); + Assert.assertTrue(S3Utils.isAwsDirectoryBucket(bucket, "")); + Assert.assertFalse(S3Utils.isAwsDirectoryBucket( + bucket, "https://minio.example.com")); + Assert.assertFalse(S3Utils.isAwsDirectoryBucket( + "analytics--x-s3", "https://s3.us-east-1.amazonaws.com")); + } +} diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index 055331c269c4de..cb5649dd9ba310 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -54,6 +54,12 @@ under the License. ${project.version} + + ${project.groupId} + fe-filesystem-api + ${project.version} + + ${project.groupId} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java index 2baae9a79926d3..f9eb7c0a85c679 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java @@ -27,6 +27,7 @@ import org.apache.doris.connector.api.scan.ConnectorScanRangeType; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsPartitionInfo; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -225,6 +226,7 @@ private void listAndSplitFiles(Configuration conf, PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize, List ranges) throws IOException { + rejectUnsupportedDirectoryBucket(partition.location, catalogProperties); Path partPath = new Path(partition.location); FileSystem fs = FileSystem.get(partPath.toUri(), conf); FileStatus[] statuses; @@ -249,6 +251,17 @@ private void listAndSplitFiles(Configuration conf, } } + static void rejectUnsupportedDirectoryBucket(String location, + Map properties) throws IOException { + String endpoint = properties.getOrDefault("AWS_ENDPOINT", + properties.getOrDefault("s3.endpoint", + properties.getOrDefault("fs.s3a.endpoint", properties.get("endpoint")))); + if (S3BucketCapabilities.isDirectoryBucketUri(location, endpoint)) { + throw new IOException( + "S3 Express One Zone is not supported by Hive Hadoop S3A in this release"); + } + } + /** * Splits a file into scan ranges based on target split size. */ diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java new file mode 100644 index 00000000000000..a5ad0401adf6c3 --- /dev/null +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.hive; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class HiveScanPlanProviderTest { + + @Test + void rejectsAwsDirectoryBucketBeforeHadoopFileSystemAccess() { + assertThrows(IOException.class, () -> HiveScanPlanProvider.rejectUnsupportedDirectoryBucket( + "s3a://analytics--use1-az4--x-s3/table", Collections.emptyMap())); + } + + @Test + void doesNotClassifyCustomS3CompatibleBucketAsDirectoryBucket() { + assertDoesNotThrow(() -> HiveScanPlanProvider.rejectUnsupportedDirectoryBucket( + "s3a://analytics--use1-az4--x-s3/table", + Map.of("fs.s3a.endpoint", "https://minio.example.com"))); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 31de8d1313e404..d733b47525fb6d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -22,6 +22,7 @@ import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.DatasourcePrintableMap; import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.ObjFileSystem; import org.apache.doris.filesystem.spi.ObjStorage; @@ -105,12 +106,24 @@ protected void setProperties(ImmutableMap newProperties) throws // the endpoint for ping need add uri scheme. String pingEndpoint = properties.get(S3Properties.ENDPOINT); if (!pingEndpoint.startsWith("http://") && !pingEndpoint.startsWith("https://")) { - pingEndpoint = "http://" + properties.get(S3Properties.ENDPOINT); + S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( + properties.get(S3Properties.BUCKET), pingEndpoint); + pingEndpoint = (capabilities.isDirectoryBucket() ? "https://" : "http://") + + properties.get(S3Properties.ENDPOINT); properties.put(S3Properties.ENDPOINT, pingEndpoint); properties.put(S3Properties.Env.ENDPOINT, pingEndpoint); } String region = S3Properties.getRegionOfEndpoint(pingEndpoint); properties.putIfAbsent(S3Properties.REGION, region); + try { + S3BucketCapabilities.resolve(properties.get(S3Properties.BUCKET), pingEndpoint) + .validateDirectoryConfiguration(pingEndpoint, + properties.get(S3Properties.REGION), + Boolean.parseBoolean(properties.getOrDefault( + S3Properties.USE_PATH_STYLE, "false"))); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } if (needCheck) { String bucketName = properties.get(S3Properties.BUCKET); @@ -166,9 +179,20 @@ protected static void pingS3(String bucketName, String rootPath, Map expectedKey.equals(object.getKey())); + continuationToken = remoteObjects.isTruncated() + ? remoteObjects.getContinuationToken() : null; + } while (!found && continuationToken != null); + if (!found) { + throw new IOException("ListObjects did not return the ping object"); + } } catch (IOException e) { throw new DdlException("pingS3 failed(list)," + " please check your endpoint, ak/sk or permissions" @@ -187,8 +211,8 @@ protected static void pingS3(String bucketName, String rootPath, Map(newProperties, "=", true, false, true, false)); + + new DatasourcePrintableMap<>(newProperties, "=", true, false, true, false), e); } try { @@ -328,4 +352,3 @@ protected void getProcNodeData(BaseProcResult result) { readUnlock(); } } - diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index 1b3158b561a7d2..abba037a641822 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -27,6 +27,7 @@ import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand; import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; import org.apache.doris.rpc.BackendServiceProxy; @@ -149,6 +150,12 @@ private void updateDefaultStorageVaultCache(Pair newDefaultVault private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map properties, String name) throws Exception { Cloud.ObjectStoreInfoPB.Builder objBuilder = S3Properties.getObjStoreInfoPB(properties); + if (objBuilder.hasProvider() && objBuilder.getProvider() == Cloud.ObjectStoreInfoPB.Provider.S3 + && S3BucketCapabilities.resolve(objBuilder.getBucket(), objBuilder.getEndpoint()) + .isDirectoryBucket()) { + throw new DdlException( + "S3 Express One Zone is not supported by Cloud Storage Vault in this release"); + } Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder(); alterObjVaultBuilder.setName(name); alterObjVaultBuilder.setObjInfo(objBuilder.build()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java index 63ab2081110409..1f24f9207fc9d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java @@ -18,6 +18,7 @@ package org.apache.doris.common.util; import org.apache.doris.common.UserException; +import org.apache.doris.filesystem.S3BucketCapabilities; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; @@ -76,9 +77,6 @@ public class S3URI { private static final Set OS_SCHEMES = ImmutableSet.of("s3", "s3a", "s3n", "bos", "oss", "cos", "cosn", "gs", "obs", "azure"); - /** Suffix of S3Express storage bucket names. */ - private static final String S3_DIRECTORY_BUCKET_SUFFIX = "--x-s3"; - private URI uri; private String bucket; @@ -356,28 +354,7 @@ public boolean useS3DirectoryBucket() { * @return true if the bucket name indicates the bucket is a directory bucket */ public static boolean isS3DirectoryBucket(final String bucketName) { - if (bucketName == null || !bucketName.endsWith(S3_DIRECTORY_BUCKET_SUFFIX)) { - return false; - } - // Check if the bucket name has the correct format: bucket-name--azid--x-s3 - // The bucket name should have at least 3 segments separated by "--" - String[] segments = bucketName.split("--"); - if (segments.length < 3) { - return false; - } - // The last segment should be "x-s3" - if (!"x-s3".equals(segments[segments.length - 1])) { - return false; - } - // The second-to-last segment should be the availability zone identifier - // It should have a format like "usw2-az1", "use1-az4", etc. - String azid = segments[segments.length - 2]; - if (azid == null || azid.isEmpty()) { - return false; - } - // Basic validation: azid should contain at least one hyphen and not be empty - // More sophisticated validation could be added here if needed - return azid.contains("-") && azid.length() > 3; + return S3BucketCapabilities.isDirectoryBucketName(bucketName); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index 3e4f4e7a62f9f6..c8eb72958bb4a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -22,6 +22,7 @@ import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.UserException; import org.apache.doris.common.credentials.CloudCredential; +import org.apache.doris.filesystem.S3BucketCapabilities; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -47,6 +48,7 @@ import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; @@ -91,37 +93,8 @@ private static AwsCredentialsProvider getAwsCredencialsProvider(CloudCredential @Deprecated public static S3Client buildS3Client(URI endpoint, String region, CloudCredential credential, boolean isUsePathStyle) { - EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy - .builder() - .baseDelay(Duration.ofSeconds(1)) - .maxBackoffTime(Duration.ofMinutes(1)) - .build(); - // retry 3 time with Equal backoff - RetryPolicy retryPolicy = RetryPolicy - .builder() - .numRetries(3) - .backoffStrategy(backoffStrategy) - .build(); - ClientOverrideConfiguration clientConf = ClientOverrideConfiguration - .builder() - // set retry policy - .retryPolicy(retryPolicy) - // using AwsS3V4Signer - .putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()) - .build(); - return S3Client.builder() - .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) - .connectionTimeout(Duration.ofSeconds(30)).build()) - .endpointOverride(endpoint) - .credentialsProvider(getAwsCredencialsProvider(credential)) - .region(Region.of(region)) - .overrideConfiguration(clientConf) - // disable chunkedEncoding because of bos not supported - .serviceConfiguration(S3Configuration.builder() - .chunkedEncodingEnabled(false) - .pathStyleAccessEnabled(isUsePathStyle) - .build()) - .build(); + return buildS3Client(endpoint, region, isUsePathStyle, + getAwsCredencialsProvider(credential), null); } /** @@ -217,6 +190,15 @@ private static AwsCredentialsProvider getAwsCredencialsProvider(URI endpoint, St public static S3Client buildS3Client(URI endpoint, String region, boolean isUsePathStyle, AwsCredentialsProvider credential) { + return buildS3Client(endpoint, region, isUsePathStyle, credential, null); + } + + public static S3Client buildS3Client(URI endpoint, String region, boolean isUsePathStyle, + AwsCredentialsProvider credential, String bucket) { + S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( + bucket, endpoint == null ? null : endpoint.toString()); + capabilities.validateDirectoryConfiguration( + endpoint == null ? null : endpoint.toString(), region, isUsePathStyle); EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy .builder() .baseDelay(Duration.ofSeconds(1)) @@ -228,62 +210,36 @@ public static S3Client buildS3Client(URI endpoint, String region, boolean isUseP .numRetries(3) .backoffStrategy(backoffStrategy) .build(); - ClientOverrideConfiguration clientConf = ClientOverrideConfiguration - .builder() - // set retry policy - .retryPolicy(retryPolicy) - // using AwsS3V4Signer - .putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()) - .build(); - return S3Client.builder() + ClientOverrideConfiguration.Builder clientConf = ClientOverrideConfiguration + .builder().retryPolicy(retryPolicy); + if (!capabilities.isDirectoryBucket()) { + clientConf.putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()); + } + S3ClientBuilder builder = S3Client.builder() .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) .connectionTimeout(Duration.ofSeconds(30)).build()) - .endpointOverride(endpoint) .credentialsProvider(credential) .region(Region.of(region)) - .overrideConfiguration(clientConf) + .disableS3ExpressSessionAuth(!capabilities.isDirectoryBucket() + && !capabilities.officialAwsService()) + .overrideConfiguration(clientConf.build()) // disable chunkedEncoding because of bos not supported .serviceConfiguration(S3Configuration.builder() .chunkedEncodingEnabled(false) - .pathStyleAccessEnabled(isUsePathStyle) - .build()) - .build(); + .pathStyleAccessEnabled(capabilities.isDirectoryBucket() + ? false : isUsePathStyle) + .build()); + if (!capabilities.isDirectoryBucket() && endpoint != null) { + builder.endpointOverride(endpoint); + } + return builder.build(); } public static S3Client buildS3Client(URI endpoint, String region, boolean isUsePathStyle, String accessKey, String secretKey, String sessionToken, String roleArn, String externalId) { - EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy - .builder() - .baseDelay(Duration.ofSeconds(1)) - .maxBackoffTime(Duration.ofMinutes(1)) - .build(); - // retry 3 time with Equal backoff - RetryPolicy retryPolicy = RetryPolicy - .builder() - .numRetries(3) - .backoffStrategy(backoffStrategy) - .build(); - ClientOverrideConfiguration clientConf = ClientOverrideConfiguration - .builder() - // set retry policy - .retryPolicy(retryPolicy) - // using AwsS3V4Signer - .putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()) - .build(); - return S3Client.builder() - .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) - .connectionTimeout(Duration.ofSeconds(30)).build()) - .endpointOverride(endpoint) - .credentialsProvider(getAwsCredencialsProvider(endpoint, region, accessKey, secretKey, - sessionToken, roleArn, externalId)) - .region(Region.of(region)) - .overrideConfiguration(clientConf) - // disable chunkedEncoding because of bos not supported - .serviceConfiguration(S3Configuration.builder() - .chunkedEncodingEnabled(false) - .pathStyleAccessEnabled(isUsePathStyle) - .build()) - .build(); + return buildS3Client(endpoint, region, isUsePathStyle, + getAwsCredencialsProvider(endpoint, region, accessKey, secretKey, + sessionToken, roleArn, externalId), null); } public static String getLongestPrefix(String globPattern) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java index 551448dc767a72..5ec5e91a33ac03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java @@ -65,7 +65,7 @@ public void testFeConnection() throws Exception { URI.create(endpoint), properties.getRegion(), Boolean.parseBoolean(properties.getUsePathStyle()), - properties.getAwsCredentialsProvider())) { + properties.getAwsCredentialsProvider(), bucket)) { client.headBucket(b -> b.bucket(bucket)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java index 0fa6a21f59efc3..8ac706ad1336fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java @@ -1869,8 +1869,18 @@ private void objCommit(Executor fileSystemExecutor, List> a + "/" + s3MPUPendingUpload.getKey(); try { objFs.completeMultipartUpload(remotePath, s3MPUPendingUpload.getUploadId(), - s3MPUPendingUpload.getEtags()); + s3MPUPendingUpload.getEtags(), + s3MPUPendingUpload.isSetChecksumAlgorithm() + ? s3MPUPendingUpload.getChecksumAlgorithm().name() : null, + s3MPUPendingUpload.isSetPartChecksums() + ? s3MPUPendingUpload.getPartChecksums() : java.util.Map.of()); } catch (java.io.IOException e) { + try { + objFs.getObjStorage().abortMultipartUpload( + remotePath, s3MPUPendingUpload.getUploadId()); + } catch (java.io.IOException abortFailure) { + e.addSuppressed(abortFailure); + } throw new RuntimeException("Failed to complete MPU for " + remotePath, e); } uncompletedMpuPendingUploads.remove(new UncompletedMpuPendingUpload(s3MPUPendingUpload, path)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index e7c6f9a64a0920..9b4fdbab2598d9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -40,6 +40,7 @@ import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -173,9 +174,18 @@ protected void doInitialize() throws UserException { long tableMetaStartTime = System.currentTimeMillis(); try { + String tableLocation = hmsTable.getRemoteTable().getSd().getLocation(); + Map storageProperties = hmsTable.getStoragePropertiesMap(); + String endpoint = storageProperties.getOrDefault("AWS_ENDPOINT", + storageProperties.getOrDefault("s3.endpoint", + storageProperties.get("fs.s3a.endpoint"))); + if (S3BucketCapabilities.isDirectoryBucketUri(tableLocation, endpoint)) { + throw new UserException( + "S3 Express One Zone is not supported by Hudi Hadoop S3A in this release"); + } hudiClient = hmsTable.getHudiClient(); hudiClient.reloadActiveTimeline(); - basePath = hmsTable.getRemoteTable().getSd().getLocation(); + basePath = tableLocation; inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); serdeLib = hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java index 2662dc55ff7fd5..0aeae737c875d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java @@ -25,6 +25,7 @@ import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.lakesoul.LakeSoulExternalTable; import org.apache.doris.datasource.lakesoul.LakeSoulUtils; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; @@ -94,6 +95,14 @@ protected void doInitialize() throws UserException { lakeSoulExternalTable = (LakeSoulExternalTable) desc.getTable(); TableInfo tableInfo = lakeSoulExternalTable.getLakeSoulTableInfo(); location = tableInfo.getTablePath(); + Map catalogProperties = lakeSoulExternalTable.getCatalog().getProperties(); + String endpoint = catalogProperties.getOrDefault("AWS_ENDPOINT", + catalogProperties.getOrDefault("s3.endpoint", + catalogProperties.get("fs.s3a.endpoint"))); + if (S3BucketCapabilities.isDirectoryBucketUri(location, endpoint)) { + throw new UserException( + "S3 Express One Zone is not supported by LakeSoul object storage in this release"); + } tableName = tableInfo.getTableName(); partitions = tableInfo.getPartitions(); readType = LakeSoulOptions.ReadType$.MODULE$.FULL_READ(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index 9c9631a516fdac..dcbfff104adf3c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -18,7 +18,9 @@ package org.apache.doris.datasource.property.metastore; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.foundation.property.ConnectorProperty; import com.google.common.collect.ImmutableList; @@ -60,6 +62,19 @@ protected AbstractPaimonProperties(Map props) { public abstract Catalog initializeCatalog(String catalogName, List storagePropertiesList); + protected void rejectDirectoryBucketStorage(List storagePropertiesList) { + String endpoint = storagePropertiesList.stream() + .filter(S3Properties.class::isInstance) + .map(S3Properties.class::cast) + .map(S3Properties::getEndpoint) + .findFirst() + .orElse(""); + if (S3BucketCapabilities.isDirectoryBucketUri(warehouse, endpoint)) { + throw new IllegalArgumentException( + "S3 Express One Zone is not supported by Paimon storage in this release"); + } + } + protected void appendCatalogOptions() { if (StringUtils.isNotBlank(warehouse)) { catalogOptions.set(CatalogOptions.WAREHOUSE.key(), warehouse); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java index 9bc77d543d3d59..74d008b499379e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java @@ -85,6 +85,7 @@ private HiveConf buildHiveConf() { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { + rejectDirectoryBucketStorage(storagePropertiesList); HiveConf hiveConf = buildHiveConf(); buildCatalogOptions(); StorageProperties ossProps = storagePropertiesList.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java index df0ebae97490a8..ccb8d778a83a52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java @@ -38,6 +38,7 @@ protected PaimonFileSystemMetaStoreProperties(Map props) { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { + rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); Configuration conf = new Configuration(); storagePropertiesList.forEach(storageProperties -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java index e7e6689d3e3cab..048ac15a48db5c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java @@ -87,6 +87,7 @@ private Configuration buildHiveConfiguration(List storageProp @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { + rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); Configuration conf = buildHiveConfiguration(storagePropertiesList); appendUserHadoopConfig(conf); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java index 7568d59c5fed33..3472f977051196 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java @@ -110,6 +110,7 @@ protected void checkRequiredProperties() { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { + rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); Configuration conf = new Configuration(); for (StorageProperties storageProperties : storagePropertiesList) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java index 465fc873b7c5d5..3a1c410b6371e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java @@ -77,6 +77,7 @@ public String getPaimonCatalogType() { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { + rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); CatalogContext catalogContext = CatalogContext.create(catalogOptions); return CatalogFactory.createCatalog(catalogContext); diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java new file mode 100644 index 00000000000000..8150841283740c --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java @@ -0,0 +1,277 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; +import java.util.regex.Pattern; + +/** Request-level S3 capabilities derived from the bucket and service endpoint. */ +public final class S3BucketCapabilities { + + public enum BucketType { + GENERAL_PURPOSE, + DIRECTORY + } + + public enum EndpointMode { + EXPLICIT_OVERRIDE, + AWS_SDK_RULES + } + + public enum ChecksumPolicy { + CONTENT_MD5, + CRC32C + } + + private static final Pattern DIRECTORY_BUCKET_PATTERN = Pattern.compile( + "^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9]+-az[0-9]+--x-s3$"); + + private final BucketType bucketType; + private final EndpointMode endpointMode; + private final ChecksumPolicy checksumPolicy; + private final boolean officialAwsService; + private final String directoryBucketZone; + + private S3BucketCapabilities(BucketType bucketType, EndpointMode endpointMode, + ChecksumPolicy checksumPolicy, boolean officialAwsService, + String directoryBucketZone) { + this.bucketType = bucketType; + this.endpointMode = endpointMode; + this.checksumPolicy = checksumPolicy; + this.officialAwsService = officialAwsService; + this.directoryBucketZone = directoryBucketZone; + } + + public static S3BucketCapabilities resolve(String bucket, String endpoint) { + boolean officialAws = isAwsS3Endpoint(endpoint); + if (!isDirectoryBucketName(bucket) || !officialAws) { + return new S3BucketCapabilities(BucketType.GENERAL_PURPOSE, + EndpointMode.EXPLICIT_OVERRIDE, ChecksumPolicy.CONTENT_MD5, officialAws, ""); + } + EndpointMode mode = isExplicitAwsEndpoint(endpoint) + ? EndpointMode.EXPLICIT_OVERRIDE : EndpointMode.AWS_SDK_RULES; + return new S3BucketCapabilities(BucketType.DIRECTORY, mode, + ChecksumPolicy.CRC32C, true, bucketZone(bucket)); + } + + public static boolean isDirectoryBucketName(String bucket) { + return bucket != null && bucket.length() >= 3 && bucket.length() <= 63 + && DIRECTORY_BUCKET_PATTERN.matcher(bucket).matches(); + } + + /** Returns true when an S3-style URI targets an AWS Directory Bucket. */ + public static boolean isDirectoryBucketUri(String location, String endpoint) { + if (location == null || location.isBlank()) { + return false; + } + try { + URI uri = new URI(location); + String scheme = uri.getScheme(); + if (scheme == null || !(scheme.equalsIgnoreCase("s3") + || scheme.equalsIgnoreCase("s3a") || scheme.equalsIgnoreCase("s3n"))) { + return false; + } + String bucket = uri.getHost() == null ? uri.getAuthority() : uri.getHost(); + return resolve(bucket, endpoint).isDirectoryBucket(); + } catch (URISyntaxException e) { + return false; + } + } + + public static boolean isAwsS3Endpoint(String endpoint) { + if (endpoint == null || endpoint.isBlank()) { + // No override means that the AWS SDK resolves the public AWS endpoint from region. + return true; + } + String host = endpointHost(endpoint); + if (host.isEmpty()) { + return false; + } + boolean awsDns = host.endsWith(".amazonaws.com") + || host.endsWith(".amazonaws.com.cn") || host.endsWith(".api.aws"); + if (!awsDns) { + return false; + } + return host.startsWith("s3.") || host.startsWith("s3-") + || host.startsWith("s3express-") || host.startsWith("s3express-control.") + || host.contains(".s3.") || host.contains(".s3-") + || host.contains(".s3express-"); + } + + public static boolean isS3ExpressControlEndpoint(String endpoint) { + String host = endpointHost(endpoint); + return host.startsWith("s3express-control.") || host.contains(".s3express-control."); + } + + public static boolean isS3ExpressZonalEndpoint(String endpoint) { + if (isS3ExpressControlEndpoint(endpoint)) { + return false; + } + String host = endpointHost(endpoint); + return host.startsWith("s3express-") || host.contains(".s3express-"); + } + + public void validateDirectoryConfiguration(String endpoint, String region, boolean usePathStyle) { + if (officialAwsService && isS3ExpressControlEndpoint(endpoint)) { + throw new IllegalArgumentException( + "s3express-control endpoint cannot be used for object operations"); + } + if (officialAwsService && isS3ExpressZonalEndpoint(endpoint) && !isDirectoryBucket()) { + throw new IllegalArgumentException( + "An S3 Express endpoint requires a valid AWS Directory Bucket name"); + } + if (!isDirectoryBucket()) { + return; + } + if (region == null || region.isBlank()) { + throw new IllegalArgumentException("AWS Directory Bucket requires AWS_REGION"); + } + if (usePathStyle) { + throw new IllegalArgumentException( + "Path-style addressing is not supported for AWS Directory Bucket"); + } + if (endpoint != null && endpoint.toLowerCase(Locale.ROOT).startsWith("http://")) { + throw new IllegalArgumentException("AWS Directory Bucket requires HTTPS"); + } + if (endpointMode != EndpointMode.AWS_SDK_RULES) { + throw new IllegalArgumentException( + "AWS Directory Bucket requires a standard regional or zonal S3 endpoint"); + } + String endpointRegion = endpointRegion(endpoint); + if (!endpointRegion.isEmpty() && !endpointRegion.equals(region)) { + throw new IllegalArgumentException("Configured endpoint region " + endpointRegion + + " does not match AWS_REGION " + region + " for Directory Bucket"); + } + String endpointZone = endpointZone(endpoint); + if (!endpointZone.isEmpty() && !endpointZone.equals(directoryBucketZone)) { + throw new IllegalArgumentException("Configured endpoint zone " + endpointZone + + " does not match Directory Bucket zone " + directoryBucketZone); + } + } + + public BucketType bucketType() { + return bucketType; + } + + public EndpointMode endpointMode() { + return endpointMode; + } + + public ChecksumPolicy checksumPolicy() { + return checksumPolicy; + } + + public boolean officialAwsService() { + return officialAwsService; + } + + public boolean isDirectoryBucket() { + return bucketType == BucketType.DIRECTORY; + } + + public boolean supportsStartAfter() { + return !isDirectoryBucket(); + } + + public boolean listIsLexicographic() { + return !isDirectoryBucket(); + } + + public boolean supportsVersioning() { + return !isDirectoryBucket(); + } + + public boolean supportsPresign() { + return !isDirectoryBucket(); + } + + private static boolean isExplicitAwsEndpoint(String endpoint) { + String host = endpointHost(endpoint); + return host.contains("fips") || host.contains("dualstack") + || host.contains("accelerate") || host.contains("vpce") + || isS3ExpressControlEndpoint(endpoint); + } + + private static String bucketZone(String bucket) { + if (bucket == null || !bucket.endsWith("--x-s3")) { + return ""; + } + int separator = bucket.lastIndexOf("--", bucket.length() - "--x-s3".length() - 1); + return separator < 0 ? "" : bucket.substring(separator + 2, bucket.length() - "--x-s3".length()); + } + + private static String endpointZone(String endpoint) { + String host = endpointHost(endpoint); + int marker = host.indexOf("s3express-"); + if (marker < 0 || host.startsWith("s3express-control", marker)) { + return ""; + } + int begin = marker + "s3express-".length(); + int end = host.indexOf('.', begin); + return end < 0 ? host.substring(begin) : host.substring(begin, end); + } + + private static String endpointRegion(String endpoint) { + String host = endpointHost(endpoint); + if (host.isEmpty() || "s3.amazonaws.com".equals(host)) { + return ""; + } + int control = host.indexOf("s3express-control."); + if (control >= 0) { + return nextHostLabel(host, control + "s3express-control.".length()); + } + int express = host.indexOf("s3express-"); + if (express >= 0) { + int dot = host.indexOf('.', express); + return dot < 0 ? "" : nextHostLabel(host, dot + 1); + } + int standard = host.indexOf("s3."); + if (standard >= 0) { + int begin = standard + "s3.".length(); + if (host.startsWith("dualstack.", begin)) { + begin += "dualstack.".length(); + } + String value = nextHostLabel(host, begin); + return "amazonaws".equals(value) ? "" : value; + } + if (host.startsWith("s3-")) { + return nextHostLabel(host, "s3-".length()); + } + return ""; + } + + private static String nextHostLabel(String host, int begin) { + int end = host.indexOf('.', begin); + return end < 0 ? host.substring(begin) : host.substring(begin, end); + } + + private static String endpointHost(String endpoint) { + if (endpoint == null || endpoint.isBlank()) { + return ""; + } + String value = endpoint.contains("://") ? endpoint : "https://" + endpoint; + try { + String host = new URI(value).getHost(); + return host == null ? "" : host.toLowerCase(Locale.ROOT); + } catch (URISyntaxException e) { + return ""; + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java index 9b2d88dc0776f6..77293bc5265013 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java @@ -27,10 +27,16 @@ public final class UploadPartResult { private final int partNumber; private final String etag; + private final String checksumCrc32c; public UploadPartResult(int partNumber, String etag) { + this(partNumber, etag, null); + } + + public UploadPartResult(int partNumber, String etag, String checksumCrc32c) { this.partNumber = partNumber; this.etag = etag; + this.checksumCrc32c = checksumCrc32c; } public int partNumber() { @@ -40,4 +46,8 @@ public int partNumber() { public String etag() { return etag; } + + public String checksumCrc32c() { + return checksumCrc32c; + } } diff --git a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java new file mode 100644 index 00000000000000..4f51be7104f808 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class S3BucketCapabilitiesTest { + + @Test + void resolvesOfficialDirectoryBucket() { + S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( + "analytics--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com"); + + Assertions.assertTrue(capabilities.isDirectoryBucket()); + Assertions.assertTrue(capabilities.officialAwsService()); + Assertions.assertEquals(S3BucketCapabilities.EndpointMode.AWS_SDK_RULES, + capabilities.endpointMode()); + Assertions.assertEquals(S3BucketCapabilities.ChecksumPolicy.CRC32C, + capabilities.checksumPolicy()); + Assertions.assertFalse(capabilities.supportsStartAfter()); + Assertions.assertFalse(capabilities.listIsLexicographic()); + Assertions.assertFalse(capabilities.supportsVersioning()); + Assertions.assertFalse(capabilities.supportsPresign()); + } + + @Test + void keepsCustomEndpointGeneralPurpose() { + S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( + "analytics--use1-az4--x-s3", "https://minio.example.com"); + + Assertions.assertFalse(capabilities.isDirectoryBucket()); + Assertions.assertFalse(capabilities.officialAwsService()); + Assertions.assertEquals(S3BucketCapabilities.ChecksumPolicy.CONTENT_MD5, + capabilities.checksumPolicy()); + } + + @Test + void rejectsUnsupportedDirectoryConfiguration() { + S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( + "analytics--use1-az4--x-s3", "https://s3.dualstack.us-east-1.amazonaws.com"); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> capabilities.validateDirectoryConfiguration( + "https://s3.dualstack.us-east-1.amazonaws.com", "us-east-1", false)); + } + + @Test + void validatesRegionAndZone() { + S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( + "analytics--use1-az4--x-s3", + "https://analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"); + + Assertions.assertDoesNotThrow(() -> capabilities.validateDirectoryConfiguration( + "https://analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com", + "us-east-1", false)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> capabilities.validateDirectoryConfiguration( + "https://analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com", + "us-west-2", false)); + + S3BucketCapabilities wrongZone = S3BucketCapabilities.resolve( + "analytics--use1-az4--x-s3", + "https://analytics--use1-az4--x-s3.s3express-use1-az5.us-east-1.amazonaws.com"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> wrongZone.validateDirectoryConfiguration( + "https://analytics--use1-az4--x-s3.s3express-use1-az5.us-east-1.amazonaws.com", + "us-east-1", false)); + } + + @Test + void recognizesSdkRoutedDirectoryBucketUri() { + Assertions.assertTrue(S3BucketCapabilities.isDirectoryBucketUri( + "s3a://analytics--use1-az4--x-s3/warehouse", "")); + Assertions.assertFalse(S3BucketCapabilities.isDirectoryBucketUri( + "s3a://analytics--use1-az4--x-s3/warehouse", "https://minio.example.com")); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java index 9ae077801c4f91..52620fa8140922 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java @@ -23,6 +23,7 @@ import org.apache.doris.filesystem.FileIterator; import org.apache.doris.filesystem.GlobListing; import org.apache.doris.filesystem.Location; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.spi.HadoopAuthenticator; import org.apache.hadoop.conf.Configuration; @@ -79,6 +80,13 @@ private org.apache.hadoop.fs.FileSystem getHadoopFs(Path path) throws IOExceptio if (closed.get()) { throw new IOException("DFSFileSystem is closed."); } + String endpoint = properties.getOrDefault("AWS_ENDPOINT", + properties.getOrDefault("s3.endpoint", + properties.getOrDefault("fs.s3a.endpoint", properties.get("endpoint")))); + if (S3BucketCapabilities.isDirectoryBucketUri(path.toString(), endpoint)) { + throw new IOException( + "S3 Express One Zone is not supported by Hadoop S3A in this release"); + } String authority = path.toUri().getAuthority(); String key = authority != null ? authority : ""; org.apache.hadoop.fs.FileSystem fs = fsByAuthority.get(key); diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java index 54abb2974acf54..e7d390004ae71e 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java @@ -69,6 +69,29 @@ void constructor_succeedsWithHdfsProperties() { Assertions.assertDoesNotThrow(() -> new DFSFileSystem(props)); } + @Test + void rejectsDirectoryBucketBeforeHadoopS3aAccess() { + DFSFileSystem fs = new DFSFileSystem(new HashMap<>()); + + IOException exception = Assertions.assertThrows(IOException.class, + () -> fs.requireFs(new Path( + "s3a://analytics--use1-az4--x-s3/warehouse/table"))); + Assertions.assertTrue(exception.getMessage().contains("Hadoop S3A")); + } + + @Test + void keepsCustomS3CompatibleBucketOnHadoopPath() throws Exception { + Map properties = Map.of( + "fs.s3a.endpoint", "https://minio.example.com"); + DFSFileSystem fs = new DFSFileSystem(properties); + org.apache.hadoop.fs.FileSystem hadoopFs = + Mockito.mock(org.apache.hadoop.fs.FileSystem.class); + injectHadoopFs(fs, "analytics--use1-az4--x-s3", hadoopFs); + + Assertions.assertSame(hadoopFs, fs.requireFs(new Path( + "s3a://analytics--use1-az4--x-s3/warehouse/table"))); + } + // ------------------------------------------------------------------ // close() and post-close behavior // ------------------------------------------------------------------ diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java index 0f6eeb35608d97..d9ddf2368f23ac 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java @@ -17,10 +17,20 @@ package org.apache.doris.filesystem.s3; +import org.apache.doris.filesystem.FileEntry; +import org.apache.doris.filesystem.GlobListing; +import org.apache.doris.filesystem.Location; +import org.apache.doris.filesystem.spi.ObjectStorageUri; +import org.apache.doris.filesystem.spi.RemoteObject; +import org.apache.doris.filesystem.spi.RemoteObjects; import org.apache.doris.filesystem.spi.S3CompatibleFileSystem; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.regex.Pattern; /** * S3 filesystem backed by the AWS S3 SDK. @@ -28,6 +38,7 @@ public class S3FileSystem extends S3CompatibleFileSystem { private final S3FileSystemProperties properties; + private final S3ObjStorage s3Storage; public S3FileSystem(S3FileSystemProperties properties) { this(properties, new S3ObjStorage(properties)); @@ -36,44 +47,83 @@ public S3FileSystem(S3FileSystemProperties properties) { S3FileSystem(S3FileSystemProperties properties, S3ObjStorage objStorage) { super(objStorage, objStorage.isUsePathStyle()); this.properties = properties; + this.s3Storage = objStorage; } public S3FileSystem(S3ObjStorage objStorage) { super(objStorage, objStorage.isUsePathStyle()); this.properties = null; + this.s3Storage = objStorage; } public Optional properties() { return Optional.ofNullable(properties); } - @Override - protected String globListPrefix(String globPattern) { - if (isDirectoryBucketEndpoint()) { - return slashTerminatedNonGlobPrefix(globPattern); + private static String slashTerminatedNonGlobPrefix(String globPattern) { + String prefix = longestNonGlobPrefix(globPattern); + if (prefix.isEmpty() || prefix.endsWith("/")) { + return prefix; } - return super.globListPrefix(globPattern); + int slash = prefix.lastIndexOf('/'); + return slash < 0 ? "" : prefix.substring(0, slash + 1); } @Override - protected List globListPrefixes(String globPattern, String listPrefix) { - if (isDirectoryBucketEndpoint()) { - return List.of(listPrefix); - } - return super.globListPrefixes(globPattern, listPrefix); + protected boolean restartListingAfterDelete(String bucket) { + return s3Storage.isDirectoryBucket(bucket); } - private boolean isDirectoryBucketEndpoint() { - return properties != null && properties.isDirectoryBucketEndpoint(); - } + @Override + public GlobListing globListWithLimit(Location path, String startAfter, long maxBytes, + long maxFiles) throws IOException { + String uri = path.uri(); + ObjectStorageUri parsed = parseUri(uri); + if (!s3Storage.isDirectoryBucket(parsed.bucket())) { + return super.globListWithLimit(path, startAfter, maxBytes, maxFiles); + } - private static String slashTerminatedNonGlobPrefix(String globPattern) { - String prefix = longestNonGlobPrefix(globPattern); - if (prefix.isEmpty() || prefix.endsWith("/")) { - return prefix; + String keyPattern = expandNumericRanges(parsed.key()); + Pattern matcher = Pattern.compile(globToRegex(keyPattern)); + String listPrefix = slashTerminatedNonGlobPrefix(keyPattern); + String base = uriBase(uri, parsed); + String listUri = base + listPrefix; + + List matches = new ArrayList<>(); + String continuationToken = null; + do { + RemoteObjects response = s3Storage.listObjects(listUri, continuationToken); + for (RemoteObject object : response.getObjectList()) { + if (!object.getKey().endsWith("/") + && matcher.matcher(object.getKey()).matches() + && (startAfter == null + || compareUtf8Binary(object.getKey(), startAfter) > 0)) { + matches.add(object); + } + } + continuationToken = response.isTruncated() + ? response.getContinuationToken() : null; + } while (continuationToken != null); + + matches.sort(Comparator.comparing(RemoteObject::getKey, S3FileSystem::compareUtf8Binary)); + List files = new ArrayList<>(); + long totalSize = 0L; + int nextIndex = matches.size(); + for (int i = 0; i < matches.size(); i++) { + RemoteObject object = matches.get(i); + files.add(new FileEntry(Location.of(base + object.getKey()), object.getSize(), false, + object.getModificationTime(), List.of())); + totalSize += object.getSize(); + if ((maxFiles > 0 && files.size() >= maxFiles) + || (maxBytes > 0 && totalSize >= maxBytes)) { + nextIndex = i + 1; + break; + } } - int slash = prefix.lastIndexOf('/'); - return slash < 0 ? "" : prefix.substring(0, slash + 1); + String maxFile = matches.isEmpty() ? "" + : nextIndex < matches.size() ? matches.get(nextIndex).getKey() + : matches.get(Math.min(files.size(), matches.size()) - 1).getKey(); + return new GlobListing(files, parsed.bucket(), listPrefix, maxFile); } protected static boolean isSingleLevelGlob(String pathStr) { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java index 34e8fb54b934bd..03a85051d59041 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java @@ -18,6 +18,7 @@ package org.apache.doris.filesystem.s3; import org.apache.doris.filesystem.FileSystemType; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.properties.BackendStorageKind; import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.FileSystemProperties; @@ -211,6 +212,8 @@ public void validate() { .check(this::hasInvalidUsePathStyle, "use_path_style must be true or false, got: '" + getUsePathStyle() + "'") .validate("Invalid S3 filesystem properties"); + capabilitiesFor(bucket).validateDirectoryConfiguration( + endpoint, region, Boolean.parseBoolean(usePathStyle)); } @Override @@ -283,6 +286,10 @@ public Map toMap() { @Override public Map toHadoopConfigurationMap() { + if (capabilitiesFor(bucket).isDirectoryBucket()) { + throw new IllegalArgumentException( + "S3 Express One Zone is not supported by Hadoop S3A in this release"); + } Map cfg = new HashMap<>(); cfg.put("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); cfg.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); @@ -327,9 +334,8 @@ public boolean hasAssumeRole() { return StringUtils.isNotBlank(roleArn); } - public boolean isDirectoryBucketEndpoint() { - return StringUtils.containsIgnoreCase(endpoint, "s3express-control.") - || StringUtils.containsIgnoreCase(endpoint, "s3express-"); + public S3BucketCapabilities capabilitiesFor(String requestBucket) { + return S3BucketCapabilities.resolve(requestBucket, endpoint); } private static void putIfNotBlank(Map map, String key, String value) { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index 274facf16529c7..7d4f484395eeb6 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -17,6 +17,7 @@ package org.apache.doris.filesystem.s3; +import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.ObjStorage; import org.apache.doris.filesystem.spi.ObjectListOptions; @@ -37,6 +38,7 @@ import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; import software.amazon.awssdk.services.s3.model.CompletedPart; @@ -98,7 +100,9 @@ public class S3ObjStorage implements ObjStorage { /** Bucket name; may be null if not provided (listObjectsWithPrefix and related methods will fail). */ private final String bucket; private final AtomicBoolean closed = new AtomicBoolean(false); - private volatile S3Client client; + private volatile S3Client generalClient; + private volatile S3Client directoryClient; + private volatile AwsCredentialsProvider credentialsProvider; public S3ObjStorage(S3FileSystemProperties properties) { this.s3Properties = properties; @@ -124,40 +128,48 @@ public S3Client getClient() throws IOException { if (closed.get()) { throw new IOException("S3ObjStorage is already closed"); } - if (client == null) { + if (generalClient == null) { synchronized (this) { - if (client == null) { - client = buildClient(); + if (generalClient == null) { + generalClient = buildClient(); } } } - return client; + return generalClient; } protected S3Client buildClient() throws IOException { return buildClient( s3Properties.getEndpoint(), s3Properties.getRegion(), - buildCredentialsProvider()); + getCredentialsProvider(), false); } - private S3Client buildClient(String endpointStr, String region, AwsCredentialsProvider credentialsProvider) + protected S3Client buildDirectoryClient() throws IOException { + return buildClient(s3Properties.getEndpoint(), s3Properties.getRegion(), + getCredentialsProvider(), true); + } + + private S3Client buildClient(String endpointStr, String region, + AwsCredentialsProvider clientCredentialsProvider, boolean directory) throws IOException { S3ClientBuilder builder = S3Client.builder() .httpClient(UrlConnectionHttpClient.builder() .socketTimeout(Duration.ofSeconds(30)) .connectionTimeout(Duration.ofSeconds(30)) .build()) - .credentialsProvider(credentialsProvider) + .credentialsProvider(clientCredentialsProvider) .region(Region.of(region)) + .disableS3ExpressSessionAuth(!directory + && !S3BucketCapabilities.isAwsS3Endpoint(endpointStr)) .serviceConfiguration(S3Configuration.builder() .chunkedEncodingEnabled(false) - .pathStyleAccessEnabled(usePathStyle) + .pathStyleAccessEnabled(directory ? false : usePathStyle) .build()); - // endpointOverride is only set for non-AWS endpoints (MinIO, COS, OSS, etc.). - // Standard AWS S3 access uses region-only routing without an explicit endpoint. - if (StringUtils.isNotBlank(endpointStr)) { + // Preserve the configured override for the general-purpose client. Directory Bucket + // requests use the separate SDK-routed client and never enter this branch. + if (!directory && StringUtils.isNotBlank(endpointStr)) { if (!endpointStr.contains("://")) { endpointStr = "https://" + endpointStr; } @@ -182,6 +194,49 @@ protected AwsCredentialsProvider buildCredentialsProvider() { return S3CredentialsProviderFactory.createClientProvider(s3Properties, this::buildStsClient); } + private AwsCredentialsProvider getCredentialsProvider() { + if (credentialsProvider == null) { + synchronized (this) { + if (credentialsProvider == null) { + credentialsProvider = buildCredentialsProvider(); + } + } + } + return credentialsProvider; + } + + S3BucketCapabilities capabilitiesFor(String requestBucket) { + return S3BucketCapabilities.resolve(requestBucket, s3Properties.getEndpoint()); + } + + boolean isDirectoryBucket(String requestBucket) { + return capabilitiesFor(requestBucket).isDirectoryBucket(); + } + + private S3Client clientFor(String requestBucket) throws IOException { + S3BucketCapabilities capabilities = capabilitiesFor(requestBucket); + if (!capabilities.isDirectoryBucket()) { + return getClient(); + } + try { + capabilities.validateDirectoryConfiguration(s3Properties.getEndpoint(), + s3Properties.getRegion(), usePathStyle); + } catch (IllegalArgumentException e) { + throw new IOException(e.getMessage(), e); + } + if (closed.get()) { + throw new IOException("S3ObjStorage is already closed"); + } + if (directoryClient == null) { + synchronized (this) { + if (directoryClient == null) { + directoryClient = buildDirectoryClient(); + } + } + } + return directoryClient; + } + private AwsCredentialsProvider buildStsSourceCredentialsProvider() { return S3CredentialsProviderFactory.createStsSourceProvider(s3Properties); } @@ -203,32 +258,42 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); + String requestPrefix = capabilities.isDirectoryBucket() + ? directoryListPrefix(uri.key()) : uri.key(); ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() .bucket(uri.bucket()) - .prefix(uri.key()); + .prefix(requestPrefix); if (options != null) { if (StringUtils.isNotBlank(options.continuationToken())) { builder.continuationToken(options.continuationToken()); - } else if (StringUtils.isNotBlank(options.startAfter())) { + } else if (capabilities.supportsStartAfter() + && StringUtils.isNotBlank(options.startAfter())) { builder.startAfter(options.startAfter()); } if (options.maxKeys() > 0) { builder.maxKeys(options.maxKeys()); } if (StringUtils.isNotBlank(options.delimiter())) { + if (capabilities.isDirectoryBucket() && !"/".equals(options.delimiter())) { + throw new IOException("AWS Directory Bucket only supports '/' as delimiter"); + } builder.delimiter(options.delimiter()); } } try { - ListObjectsV2Response response = getClient().listObjectsV2(builder.build()); - List objects = response.contents().stream() + List objects = new ArrayList<>(); + ListObjectsV2Response response = clientFor(uri.bucket()).listObjectsV2(builder.build()); + response.contents().stream() + .filter(s3Obj -> s3Obj.key().startsWith(uri.key())) .map(s3Obj -> new org.apache.doris.filesystem.spi.RemoteObject( s3Obj.key(), getRelativePath(uri.key(), s3Obj.key()), s3Obj.eTag(), s3Obj.size(), - s3Obj.lastModified() != null ? s3Obj.lastModified().toEpochMilli() : 0L)) - .collect(Collectors.toList()); + s3Obj.lastModified() != null + ? s3Obj.lastModified().toEpochMilli() : 0L)) + .forEach(objects::add); return new RemoteObjects(objects, response.isTruncated(), response.nextContinuationToken()); } catch (SdkException e) { @@ -285,7 +350,7 @@ public RemoteObjects listObjectsNonRecursive(String remotePath, String continuat public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - HeadObjectResponse response = getClient().headObject( + HeadObjectResponse response = clientFor(uri.bucket()).headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return new org.apache.doris.filesystem.spi.RemoteObject( uri.key(), uri.key(), response.eTag(), response.contentLength(), @@ -306,9 +371,15 @@ public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath public void putObject(String remotePath, org.apache.doris.filesystem.spi.RequestBody requestBody) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); + PutObjectRequest.Builder request = PutObjectRequest.builder() + .bucket(uri.bucket()).key(uri.key()); + if (capabilities.checksumPolicy() == S3BucketCapabilities.ChecksumPolicy.CRC32C) { + request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); + } try (InputStream content = requestBody.content()) { - getClient().putObject( - PutObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build(), + clientFor(uri.bucket()).putObject( + request.build(), software.amazon.awssdk.core.sync.RequestBody.fromInputStream( content, requestBody.contentLength())); } catch (SdkException e) { @@ -320,7 +391,7 @@ public void putObject(String remotePath, org.apache.doris.filesystem.spi.Request public void deleteObject(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - getClient().deleteObject(DeleteObjectRequest.builder() + clientFor(uri.bucket()).deleteObject(DeleteObjectRequest.builder() .bucket(uri.bucket()).key(uri.key()).build()); } catch (S3Exception e) { if (e.statusCode() == 404) { @@ -336,8 +407,11 @@ public void deleteObject(String remotePath) throws IOException { public void copyObject(String srcPath, String dstPath) throws IOException { S3Uri srcUri = S3Uri.parse(srcPath, usePathStyle); S3Uri dstUri = S3Uri.parse(dstPath, usePathStyle); + if (isDirectoryBucket(srcUri.bucket()) || isDirectoryBucket(dstUri.bucket())) { + throw new IOException("CopyObject is not supported for AWS Directory Bucket"); + } try { - getClient().copyObject(CopyObjectRequest.builder() + clientFor(dstUri.bucket()).copyObject(CopyObjectRequest.builder() .copySource(SdkHttpUtils.urlEncodeIgnoreSlashes( srcUri.bucket() + "/" + srcUri.key())) .destinationBucket(dstUri.bucket()) @@ -352,9 +426,15 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); + CreateMultipartUploadRequest.Builder request = CreateMultipartUploadRequest.builder() + .bucket(uri.bucket()).key(uri.key()); + if (capabilities.checksumPolicy() == S3BucketCapabilities.ChecksumPolicy.CRC32C) { + request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); + } try { - CreateMultipartUploadResponse response = getClient().createMultipartUpload( - CreateMultipartUploadRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); + CreateMultipartUploadResponse response = clientFor(uri.bucket()) + .createMultipartUpload(request.build()); return response.uploadId(); } catch (SdkException e) { throw new IOException("initiateMultipartUpload failed for " + remotePath @@ -366,16 +446,24 @@ public String initiateMultipartUpload(String remotePath) throws IOException { public UploadPartResult uploadPart(String remotePath, String uploadId, int partNum, org.apache.doris.filesystem.spi.RequestBody body) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); + S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); + UploadPartRequest.Builder request = UploadPartRequest.builder() + .bucket(uri.bucket()).key(uri.key()) + .uploadId(uploadId).partNumber(partNum) + .contentLength(body.contentLength()); + if (capabilities.checksumPolicy() == S3BucketCapabilities.ChecksumPolicy.CRC32C) { + request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); + } try (InputStream content = body.content()) { - UploadPartResponse response = getClient().uploadPart( - UploadPartRequest.builder() - .bucket(uri.bucket()).key(uri.key()) - .uploadId(uploadId).partNumber(partNum) - .contentLength(body.contentLength()) - .build(), + UploadPartResponse response = clientFor(uri.bucket()).uploadPart( + request.build(), software.amazon.awssdk.core.sync.RequestBody.fromInputStream( content, body.contentLength())); - return new UploadPartResult(partNum, response.eTag()); + if (capabilities.isDirectoryBucket() && StringUtils.isBlank(response.checksumCRC32C())) { + throw new IOException("UploadPart response is missing CRC32C for AWS Directory Bucket, part=" + + partNum); + } + return new UploadPartResult(partNum, response.eTag(), response.checksumCRC32C()); } catch (SdkException e) { throw new IOException("uploadPart " + partNum + " failed for " + remotePath + ": " + e.getMessage(), e); @@ -386,11 +474,29 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); - List completedParts = parts.stream() - .map(p -> CompletedPart.builder().partNumber(p.partNumber()).eTag(p.etag()).build()) + S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); + List sortedParts = parts.stream() + .sorted(java.util.Comparator.comparingInt(UploadPartResult::partNumber)) .collect(Collectors.toList()); + List completedParts = new ArrayList<>(sortedParts.size()); + for (int i = 0; i < sortedParts.size(); i++) { + UploadPartResult part = sortedParts.get(i); + if (capabilities.isDirectoryBucket() && part.partNumber() != i + 1) { + throw new IOException("Multipart upload parts must be consecutive from 1"); + } + CompletedPart.Builder completed = CompletedPart.builder() + .partNumber(part.partNumber()).eTag(part.etag()); + if (capabilities.isDirectoryBucket()) { + if (StringUtils.isBlank(part.checksumCrc32c())) { + throw new IOException( + "AWS Directory Bucket multipart completion requires CRC32C for every part"); + } + completed.checksumCRC32C(part.checksumCrc32c()); + } + completedParts.add(completed.build()); + } try { - getClient().completeMultipartUpload(CompleteMultipartUploadRequest.builder() + clientFor(uri.bucket()).completeMultipartUpload(CompleteMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId) .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) .build()); @@ -404,9 +510,12 @@ public void completeMultipartUpload(String remotePath, String uploadId, public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - getClient().abortMultipartUpload(AbortMultipartUploadRequest.builder() + clientFor(uri.bucket()).abortMultipartUpload(AbortMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId).build()); } catch (S3Exception e) { + if (e.statusCode() == 404) { + return; + } // Re-throw so callers know the abort failed; orphaned parts may still exist // and require manual cleanup or a lifecycle rule. throw new IOException("abortMultipartUpload failed for " + remotePath @@ -424,7 +533,7 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx InputStream openInputStream(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - return getClient().getObject( + return clientFor(uri.bucket()).getObject( GetObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); } catch (NoSuchKeyException e) { throw new FileNotFoundException("Object not found: " + remotePath); @@ -445,7 +554,7 @@ public InputStream openInputStreamAt(String remotePath, long fromByte) throws IO if (fromByte > 0) { req.range("bytes=" + fromByte + "-"); } - return getClient().getObject(req.build()); + return clientFor(uri.bucket()).getObject(req.build()); } catch (NoSuchKeyException e) { throw new FileNotFoundException("Object not found: " + remotePath); } catch (SdkException e) { @@ -460,7 +569,7 @@ public InputStream openInputStreamAt(String remotePath, long fromByte) throws IO public long headObjectLastModified(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - HeadObjectResponse resp = getClient().headObject( + HeadObjectResponse resp = clientFor(uri.bucket()).headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return resp.lastModified() != null ? resp.lastModified().toEpochMilli() : 0L; } catch (NoSuchKeyException e) { @@ -504,15 +613,19 @@ public RemoteObjects listObjectsWithPrefix(String prefix, String subPrefix, String continuationToken) throws IOException { requireBucket("listObjectsWithPrefix"); String fullPrefix = normalizeAndCombinePrefix(prefix, subPrefix); + S3BucketCapabilities capabilities = capabilitiesFor(bucket); + String requestPrefix = capabilities.isDirectoryBucket() + ? directoryListPrefix(fullPrefix) : fullPrefix; try { ListObjectsV2Request.Builder reqBuilder = ListObjectsV2Request.builder() .bucket(bucket) - .prefix(fullPrefix); + .prefix(requestPrefix); if (StringUtils.isNotBlank(continuationToken)) { reqBuilder.continuationToken(continuationToken); } - ListObjectsV2Response resp = getClient().listObjectsV2(reqBuilder.build()); + ListObjectsV2Response resp = clientFor(bucket).listObjectsV2(reqBuilder.build()); List files = resp.contents().stream() + .filter(s3Obj -> s3Obj.key().startsWith(fullPrefix)) .map(s3Obj -> new RemoteObject( s3Obj.key(), getRelativePathSafe(prefix, s3Obj.key()), @@ -533,7 +646,7 @@ public RemoteObjects headObjectWithMeta(String prefix, String subKey) throws IOE requireBucket("headObjectWithMeta"); String fullKey = normalizeAndCombinePrefix(prefix, subKey); try { - HeadObjectResponse resp = getClient().headObject( + HeadObjectResponse resp = clientFor(bucket).headObject( HeadObjectRequest.builder() .bucket(bucket) .key(fullKey) @@ -554,6 +667,9 @@ fullKey, getRelativePathSafe(prefix, fullKey), resp.eTag(), resp.contentLength() @Override public String getPresignedUrl(String objectKey) throws IOException { requireBucket("getPresignedUrl"); + if (!capabilitiesFor(bucket).supportsPresign()) { + throw new IOException("Presigned URL is not supported for AWS Directory Bucket"); + } try { PutObjectRequest putReq = PutObjectRequest.builder() .bucket(bucket) @@ -589,11 +705,14 @@ public void deleteObjectsByKeys(String bucket, List keys) throws IOExcep List identifiers = batch.stream() .map(k -> ObjectIdentifier.builder().key(k).build()) .collect(Collectors.toList()); - DeleteObjectsRequest request = DeleteObjectsRequest.builder() + DeleteObjectsRequest.Builder request = DeleteObjectsRequest.builder() .bucket(bucket) - .delete(Delete.builder().objects(identifiers).quiet(true).build()) - .build(); - DeleteObjectsResponse response = getClient().deleteObjects(request); + .delete(Delete.builder().objects(identifiers).quiet(true).build()); + if (capabilitiesFor(bucket).checksumPolicy() + == S3BucketCapabilities.ChecksumPolicy.CRC32C) { + request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); + } + DeleteObjectsResponse response = clientFor(bucket).deleteObjects(request.build()); if (response.hasErrors()) { for (S3Error error : response.errors()) { LOG.warn("Failed to delete object key={} from bucket={}: {} {}", @@ -640,6 +759,14 @@ private static String normalizeAndCombinePrefix(String prefix, String subPrefix) return normalized.isEmpty() ? subPrefix : normalized + subPrefix; } + private static String directoryListPrefix(String prefix) { + if (prefix == null || prefix.isEmpty() || prefix.endsWith("/")) { + return prefix == null ? "" : prefix; + } + int slash = prefix.lastIndexOf('/'); + return slash < 0 ? "" : prefix.substring(0, slash + 1); + } + private static String getRelativePathSafe(String prefix, String key) { String normalized = (prefix == null || prefix.isEmpty()) ? "" : (prefix.endsWith("/") ? prefix : prefix + "/"); @@ -652,10 +779,14 @@ private static String getRelativePathSafe(String prefix, String key) { @Override public void close() throws IOException { if (closed.compareAndSet(false, true)) { - if (client != null) { - client.close(); - client = null; + if (generalClient != null) { + generalClient.close(); + } + if (directoryClient != null && directoryClient != generalClient) { + directoryClient.close(); } + generalClient = null; + directoryClient = null; } } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java index e96bcf53b8e4ed..e1cf014d1b151f 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java @@ -271,4 +271,18 @@ void of_rejectsUnsupportedCredentialsProviderType() { Assertions.assertTrue(exception.getMessage().contains("Invalid S3 filesystem properties")); Assertions.assertTrue(exception.getMessage().contains("Unsupported s3.credentials_provider_type")); } + + @Test + void directoryBucketRejectsHadoopS3AConversion() { + Map raw = new HashMap<>(); + raw.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com"); + raw.put("s3.region", "us-east-1"); + raw.put("s3.bucket", "analytics--use1-az4--x-s3"); + + S3FileSystemProperties properties = S3FileSystemProperties.of(raw); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, properties::toHadoopConfigurationMap); + Assertions.assertTrue(exception.getMessage().contains("Hadoop S3A")); + } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java index fe9b28dd8c90b4..c135fbc002b0d0 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java @@ -678,28 +678,33 @@ void globListWithLimit_paginatesExpandedPrefixesInUtf8BinaryOrder() throws IOExc @Test void globListWithLimit_directoryBucketFallsBackToSlashTerminatedStaticPrefix() throws IOException { S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( - "s3.endpoint", "https://s3express-usw2-az1.us-west-2.amazonaws.com", - "s3.region", "us-west-2")); + "s3.endpoint", "https://s3.us-east-1.amazonaws.com", + "s3.region", "us-east-1")); S3FileSystem directoryBucketFs = new S3FileSystem(properties, mockStorage); + String bucket = "analytics--use1-az4--x-s3"; + Mockito.when(mockStorage.isDirectoryBucket(bucket)).thenReturn(true); Mockito.when(mockStorage.listObjects( - ArgumentMatchers.eq("s3://bucket/data/"), ArgumentMatchers.isNull())) + ArgumentMatchers.eq("s3://" + bucket + "/data/"), ArgumentMatchers.isNull())) .thenReturn(new RemoteObjects( List.of( - new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L), - new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L)), + new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L), + new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L)), false, null)); GlobListing listing = directoryBucketFs.globListWithLimit( - Location.of("s3://bucket/data/[ab]*.csv"), null, 0L, 0L); + Location.of("s3://" + bucket + "/data/[ab]*.csv"), null, 0L, 1L); - Assertions.assertEquals(2, listing.getFiles().size()); + Assertions.assertEquals(1, listing.getFiles().size()); + Assertions.assertEquals("s3://" + bucket + "/data/a.csv", + listing.getFiles().get(0).location().uri()); + Assertions.assertEquals("data/b.csv", listing.getMaxFile()); Assertions.assertEquals("data/", listing.getPrefix()); Mockito.verify(mockStorage).listObjects( - ArgumentMatchers.eq("s3://bucket/data/"), ArgumentMatchers.isNull()); + ArgumentMatchers.eq("s3://" + bucket + "/data/"), ArgumentMatchers.isNull()); Mockito.verify(mockStorage, Mockito.never()).listObjects( - ArgumentMatchers.eq("s3://bucket/data/a"), ArgumentMatchers.any()); + ArgumentMatchers.eq("s3://" + bucket + "/data/a"), ArgumentMatchers.any()); Mockito.verify(mockStorage, Mockito.never()).listObjects( - ArgumentMatchers.eq("s3://bucket/data/b"), ArgumentMatchers.any()); + ArgumentMatchers.eq("s3://" + bucket + "/data/b"), ArgumentMatchers.any()); } @Test diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java index deefd3925a5855..7f1dd761f9ab96 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java @@ -33,6 +33,7 @@ import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; @@ -91,6 +92,87 @@ void getClient_returnsInjectedMockClient() throws IOException { Assertions.assertEquals(mockS3, storage.getClient()); } + @Test + void directoryBucketUsesCrc32cForPutAndMultipart() throws IOException { + Map props = new HashMap<>(); + props.put("AWS_ENDPOINT", "https://s3.us-east-1.amazonaws.com"); + props.put("AWS_REGION", "us-east-1"); + props.put("AWS_ACCESS_KEY", "testAK"); + props.put("AWS_SECRET_KEY", "testSK"); + props.put("AWS_BUCKET", "analytics--use1-az4--x-s3"); + storage = new TestableS3ObjStorage(props, mockS3); + + Mockito.when(mockS3.putObject(ArgumentMatchers.any(PutObjectRequest.class), + ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class))) + .thenReturn(PutObjectResponse.builder().build()); + Mockito.when(mockS3.createMultipartUpload( + ArgumentMatchers.any(CreateMultipartUploadRequest.class))) + .thenReturn(CreateMultipartUploadResponse.builder().uploadId("upload-1").build()); + Mockito.when(mockS3.uploadPart(ArgumentMatchers.any(UploadPartRequest.class), + ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class))) + .thenReturn(UploadPartResponse.builder().eTag("etag-1") + .checksumCRC32C("crc32c-1").build()); + Mockito.when(mockS3.completeMultipartUpload( + ArgumentMatchers.any(CompleteMultipartUploadRequest.class))) + .thenReturn(software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse + .builder().build()); + Mockito.when(mockS3.deleteObjects(ArgumentMatchers.any( + software.amazon.awssdk.services.s3.model.DeleteObjectsRequest.class))) + .thenReturn(software.amazon.awssdk.services.s3.model.DeleteObjectsResponse + .builder().build()); + + String path = "s3://analytics--use1-az4--x-s3/data/file"; + storage.putObject(path, RequestBody.of(new ByteArrayInputStream(new byte[] {1}), 1)); + String uploadId = storage.initiateMultipartUpload(path); + UploadPartResult part = storage.uploadPart(path, uploadId, 1, + RequestBody.of(new ByteArrayInputStream(new byte[] {1}), 1)); + storage.completeMultipartUpload(path, uploadId, List.of(part)); + storage.deleteObjectsByKeys("analytics--use1-az4--x-s3", List.of("data/file")); + + ArgumentCaptor putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); + Mockito.verify(mockS3).putObject(putCaptor.capture(), + ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class)); + Assertions.assertEquals(ChecksumAlgorithm.CRC32C, putCaptor.getValue().checksumAlgorithm()); + ArgumentCaptor createCaptor = + ArgumentCaptor.forClass(CreateMultipartUploadRequest.class); + Mockito.verify(mockS3).createMultipartUpload(createCaptor.capture()); + Assertions.assertEquals(ChecksumAlgorithm.CRC32C, + createCaptor.getValue().checksumAlgorithm()); + Assertions.assertEquals("crc32c-1", part.checksumCrc32c()); + ArgumentCaptor completeCaptor = + ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class); + Mockito.verify(mockS3).completeMultipartUpload(completeCaptor.capture()); + Assertions.assertEquals("crc32c-1", completeCaptor.getValue().multipartUpload() + .parts().get(0).checksumCRC32C()); + ArgumentCaptor deleteCaptor = + ArgumentCaptor.forClass( + software.amazon.awssdk.services.s3.model.DeleteObjectsRequest.class); + Mockito.verify(mockS3).deleteObjects(deleteCaptor.capture()); + Assertions.assertEquals(ChecksumAlgorithm.CRC32C, + deleteCaptor.getValue().checksumAlgorithm()); + } + + @Test + void directoryListOmitsStartAfterAndUsesParentPrefix() throws IOException { + Map props = new HashMap<>(); + props.put("AWS_ENDPOINT", "https://s3.us-east-1.amazonaws.com"); + props.put("AWS_REGION", "us-east-1"); + props.put("AWS_ACCESS_KEY", "testAK"); + props.put("AWS_SECRET_KEY", "testSK"); + storage = new TestableS3ObjStorage(props, mockS3); + Mockito.when(mockS3.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + + storage.listObjectsWithOptions("s3://analytics--use1-az4--x-s3/data/file*", + org.apache.doris.filesystem.spi.ObjectListOptions.builder() + .startAfter("data/file0").build()); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + Mockito.verify(mockS3).listObjectsV2(captor.capture()); + Assertions.assertEquals("data/", captor.getValue().prefix()); + Assertions.assertNull(captor.getValue().startAfter()); + } + // ------------------------------------------------------------------ // listObjects() // ------------------------------------------------------------------ @@ -499,6 +581,11 @@ private static class TestableS3ObjStorage extends S3ObjStorage { protected S3Client buildClient() { return mockClient; } + + @Override + protected S3Client buildDirectoryClient() { + return mockClient; + } } private static class InspectableS3ObjStorage extends TestableS3ObjStorage { diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java index f98a9e9640d426..f5c7bae56ca525 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java @@ -109,8 +109,18 @@ public void deleteObjectsByKeys(String bucket, List keys) throws IOExcep */ public void completeMultipartUpload(String remotePath, String uploadId, Map etags) throws IOException { + completeMultipartUpload(remotePath, uploadId, etags, null, Map.of()); + } + + public void completeMultipartUpload(String remotePath, String uploadId, + Map etags, String checksumAlgorithm, + Map partChecksums) throws IOException { + if (checksumAlgorithm != null && !"CRC32C".equals(checksumAlgorithm)) { + throw new IOException("Unsupported multipart checksum algorithm: " + checksumAlgorithm); + } List parts = etags.entrySet().stream() - .map(e -> new UploadPartResult(e.getKey(), e.getValue())) + .map(e -> new UploadPartResult(e.getKey(), e.getValue(), + partChecksums.get(e.getKey()))) .sorted(Comparator.comparingInt(UploadPartResult::partNumber)) .collect(Collectors.toList()); objStorage.completeMultipartUpload(remotePath, uploadId, parts); diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java index 5746fed5dd8549..d124ef50d424b9 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java @@ -195,19 +195,28 @@ private void deleteRecursive(String prefix) throws IOException { String continuationToken = null; do { RemoteObjects batch = objStorage.listObjects(prefix, continuationToken); + boolean deletedObjects = false; if (!batch.getObjectList().isEmpty()) { List keys = new ArrayList<>(batch.getObjectList().size()); for (RemoteObject obj : batch.getObjectList()) { keys.add(obj.getKey()); } objStorage.deleteObjectsByKeys(bucket, keys); + deletedObjects = true; } - continuationToken = batch.isTruncated() ? batch.getContinuationToken() : null; + continuationToken = batch.isTruncated() + ? (restartListingAfterDelete(bucket) && deletedObjects + ? "" : batch.getContinuationToken()) + : null; } while (continuationToken != null); } + protected boolean restartListingAfterDelete(String bucket) { + return false; + } + /** Parses {@code uri} respecting the underlying client's path-style configuration. */ - private ObjectStorageUri parseUri(String uri) { + protected ObjectStorageUri parseUri(String uri) { return ObjectStorageUri.parse(uri, usePathStyle); } @@ -217,7 +226,7 @@ private ObjectStorageUri parseUri(String uri) { * Computed by stripping the parsed key off the original URI string, so it is correct * for both virtual-hosted and path-style URIs without re-deriving the host. */ - private static String uriBase(String uri, ObjectStorageUri parsed) { + protected static String uriBase(String uri, ObjectStorageUri parsed) { String key = parsed.key(); String base = key.isEmpty() ? uri : uri.substring(0, uri.length() - key.length()); return base.endsWith("/") ? base : base + "/"; @@ -1103,7 +1112,7 @@ private static List compactPrefixes(List prefixes) { return compact; } - private static int compareUtf8Binary(String left, String right) { + protected static int compareUtf8Binary(String left, String right) { byte[] leftBytes = left.getBytes(StandardCharsets.UTF_8); byte[] rightBytes = right.getBytes(StandardCharsets.UTF_8); int commonLength = Math.min(leftBytes.length, rightBytes.length); @@ -1142,7 +1151,7 @@ private PrefixExpansion(List values, int nextIndex) { *
  • {@code data_{-1..1}.csv} → unchanged (no expansion)
    • * */ - private static String expandNumericRanges(String pattern) { + protected static String expandNumericRanges(String pattern) { java.util.regex.Pattern rangeSegment = java.util.regex.Pattern.compile( "(-?\\d+)\\.\\.(-?\\d+)"); java.util.regex.Pattern simpleRange = java.util.regex.Pattern.compile( diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 3a45208ea9bf11..b7057564c56fc0 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -392,11 +392,17 @@ enum TUpdateMode { OVERWRITE = 2 // insert overwrite } +enum TObjectStorageChecksumAlgorithm { + CRC32C = 0 +} + struct TS3MPUPendingUpload { 1: optional string bucket 2: optional string key 3: optional string upload_id 4: optional map etags + 5: optional TObjectStorageChecksumAlgorithm checksum_algorithm + 6: optional map part_checksums } struct THivePartitionUpdate { From 125a1ff03b7fee6197778677fe53be3ed396fc1d Mon Sep 17 00:00:00 2001 From: 0AyanamiRei <3244156674@qq.com> Date: Thu, 16 Jul 2026 23:33:19 +0800 Subject: [PATCH 06/13] [doc](s3) Document S3 Express implementation changes ### What problem does this PR solve? Issue Number: None Related PR: #63409 Problem Summary: Document how the S3 Express One Zone design maps to the implemented BE, FE, Hive, Cloud boundary, checksum, multipart, listing, and fail-fast changes. Record compatibility behavior, unexecuted tests, and remaining validation gaps so the implementation status is not confused with GA acceptance. ### Release note None ### Check List (For Author) - Test: No need to test (documentation-only change) - Behavior changed: No - Does this need documentation: Yes, this commit adds the implementation summary --- s3_express_one_zone_implementation.md | 534 ++++++++++++++++++++++++++ 1 file changed, 534 insertions(+) create mode 100644 s3_express_one_zone_implementation.md diff --git a/s3_express_one_zone_implementation.md b/s3_express_one_zone_implementation.md new file mode 100644 index 00000000000000..21ecbc7bce034c --- /dev/null +++ b/s3_express_one_zone_implementation.md @@ -0,0 +1,534 @@ +# Apache Doris S3 Express One Zone 实现改动梳理 + +## 1. 文档目的 + +本文梳理 `goal.md` 设计方案在提交 `8f732dce8a` 中的实际落地情况，重点回答： + +- Doris 为支持 S3 Express One Zone 实际修改了哪些模块。 +- Directory Bucket 请求与普通 S3 请求分别经过什么调用链。 +- checksum、multipart、ListObjectsV2 和失败清理如何处理。 +- 哪些 Doris 使用面已经接入原生实现，哪些使用面仍然主动拒绝。 +- 当前实现已经具备哪些代码能力，以及还缺少哪些编译、测试和真实 AWS 验证。 + +本文是实现状态说明，不替代 `goal.md` 中的完整背景、AWS 约束、发布标准和运维方案。 + +## 2. 实现状态 + +| 项目 | 内容 | +| --- | --- | +| 设计文档 | `goal.md` | +| 实现提交 | `8f732dce8a`，`[feature](s3) Support S3 Express One Zone directory buckets` | +| 当前阶段 | 代码已实现，尚未编译和执行测试 | +| 原生支持面 | BE S3 FileSystem、FE Java SDK v2 S3 FileSystem、S3 Resource 连通性检查、Hive multipart 提交链路 | +| 主动拒绝面 | Cloud Storage Vault、Hadoop/S3A 外部扫描、presigned URL、Java S3 CopyObject | +| 兼容目标 | 普通 AWS S3 与 MinIO、COS、OSS 等 S3-compatible 服务保持原有行为 | + +这里的“代码已实现”表示设计中的主要调用链已经进入代码，不表示已经满足 GA 验收。当前没有执行编译、单元测试、回归测试或真实 Directory Bucket 集成测试。 + +## 3. 总体设计 + +本次实现没有把 S3 Express 处理成一个全局开关，而是引入 request-level bucket capability： + +```text +bucket + endpoint + region + addressing mode + | + v + S3BucketCapabilities resolver + | + +---------+---------+ + | | + v v + GENERAL_PURPOSE DIRECTORY + | | + endpoint override SDK endpoint rules + Content-MD5 CRC32C + ordinary list continuation + local ordering + ordinary auth SDK-managed Express session auth +``` + +Directory Bucket 只有在以下条件同时成立时才会被识别： + +1. bucket 是合法的 Availability Zone Directory Bucket 完整名称，例如 + `analytics--use1-az4--x-s3`。 +2. endpoint 为空，或者属于官方 AWS S3 endpoint。 + +因此，在 MinIO 等自定义 endpoint 下，即使 bucket 名以 `--x-s3` 结尾，也仍按普通 S3-compatible bucket 处理。 + +## 4. 公共能力模型 + +### 4.1 C++ + +新增文件： + +- `common/cpp/s3_bucket_capabilities.h` + +核心类型： + +- `S3BucketType` + - `GENERAL_PURPOSE` + - `DIRECTORY` +- `S3EndpointMode` + - `EXPLICIT_OVERRIDE` + - `AWS_SDK_RULES` +- `S3ChecksumPolicy` + - `CONTENT_MD5` + - `CRC32C` +- `S3BucketCapabilities` + - 是否要求 HTTPS。 + - 是否要求 virtual-hosted addressing。 + - 是否支持 `StartAfter`。 + - 服务端结果是否保证字典序。 + - 是否支持 versioning 和 presign。 + +resolver 同时提供以下辅助校验： + +- Directory Bucket 名称和 zone-id 解析。 +- AWS S3、S3 Express Zonal 和 control endpoint 识别。 +- endpoint 中 region、zone-id 的解析与冲突检查。 +- FIPS、dualstack、accelerate、VPC endpoint 等显式 endpoint mode 识别。 + +### 4.2 Java + +新增文件： + +- `fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java` + +Java 侧镜像 C++ 的分类和配置校验规则，供 `fe-core`、`fe-filesystem-s3`、Hadoop FileSystem 和 connector 模块复用。 + +目前 C++ 和 Java 使用相同规则并分别有测试，但尚未按 `goal.md` 的建议抽取为一份由两端共同读取的测试向量文件。这是后续防止规则漂移的改进项。 + +## 5. BE 改动 + +### 5.1 S3ClientConf 与客户端缓存 + +修改文件： + +- `be/src/util/s3_util.h` +- `be/src/util/s3_util.cpp` + +主要改动： + +1. `S3ClientConf` 增加基于 bucket capability 的 client 行为派生。 +2. cache hash 改为有顺序的 hash combine，不再对所有字段直接 XOR。 +3. cache key 补充： + - bucket + - `need_override_endpoint` + - addressing mode + - credentials provider + - role ARN、external ID、token identity + - bucket type、endpoint mode、checksum policy + - 全局 HTTP scheme +4. `to_string()` 不再输出明文 token 和 external ID。 +5. 删除全局动态配置 `s3_disable_content_md5`，checksum 完全由 bucket capability 决定。 + +由于 bucket 进入 client cache key，不同 Directory Bucket 不会复用同一个 Doris cache entry；SDK 内部产生的 bucket-scoped session credentials 也不会进入 Doris 配置、日志或 RPC。 + +### 5.2 S3ClientFactory + +普通 S3 路径保持： + +- 原有 endpoint override。 +- 原有 virtual/path-style 行为。 +- `PayloadSigningPolicy::Never`。 +- Content-MD5。 + +Directory Bucket 路径改为： + +- 从 Doris 公共 `ClientConfiguration` 转换为 `S3ClientConfiguration`，保留 CA、timeout 和 retry 配置。 +- 不设置 `endpointOverride`。 +- 使用 `S3EndpointProvider` 和 SDK endpoint rules。 +- `disableS3ExpressAuth=false`，由 SDK 完成 CreateSession、session cache 和刷新。 +- `PayloadSigningPolicy::RequestDependent`。 +- 要求 region、HTTPS 和 virtual-hosted addressing。 +- 拒绝 path-style、HTTP、control endpoint、显式 FIPS/dualstack 等不支持配置，以及 region/zone 冲突。 + +自定义 S3-compatible endpoint 使用 `disableS3ExpressAuth=true`，避免 SDK 因特殊 bucket 名误启用 Express session auth。 + +### 5.3 特殊客户端创建入口 + +`be/src/exprs/function/ai/embed.h` 调整为先解析 URI 和 bucket，再创建 S3 client。Directory Bucket 的 AI media presigned URL 在创建 client 前返回 `NotSupported`。 + +`be/src/io/tools/file_cache_microbench.cpp` 补充 bucket 到 `S3ClientConf`，使 capability 和 cache key 能正确派生。 + +### 5.4 PutObject 与 UploadPart checksum + +修改文件： + +- `be/src/io/fs/s3_obj_storage_client.h` +- `be/src/io/fs/s3_obj_storage_client.cpp` + +`S3ObjStorageClient` 不再保存 `_disable_content_md5`，而是保存完整 `S3BucketCapabilities`。 + +Directory Bucket 写入行为： + +- `PutObject` 计算 Base64 CRC32C，设置 algorithm 和 checksum，不发送 Content-MD5。 +- `CreateMultipartUpload` 声明 `CRC32C`。 +- `UploadPart` 计算并发送 CRC32C。 +- `UploadPart` 校验响应中的 CRC32C；缺失或不一致均返回失败。 +- `CompleteMultipartUpload` 为每个 part 发送 ETag、part number 和 CRC32C。 +- Complete 前要求 Directory Bucket part number 为从 1 开始的连续序列。 + +普通 S3 继续发送 Content-MD5，不使用 Directory Bucket 分支。 + +### 5.5 Multipart 元数据与失败清理 + +修改文件： + +- `be/src/io/fs/obj_storage_client.h` +- `be/src/io/fs/s3_file_writer.cpp` + +新增数据： + +- `ObjectStorageUploadResponse.checksum_crc32c` +- `ObjectCompleteMultiPart.checksum_crc32c` + +`S3FileWriter` 会保存每个 part 的 CRC32C，在 Complete 前排序并检查 part 序列。 + +失败清理流程： + +```text +CreateMultipartUpload 成功 + | + v +并发 UploadPart / Complete / close + | + +---- 成功 ----> Complete + | + +---- 失败 + | + v + 等待所有异步 part 收敛 + | + v + AbortMultipartUpload +``` + +具体行为： + +- `ObjStorageClient` 增加 `abort_multipart_upload` 接口。 +- S3 实现调用 AWS `AbortMultipartUpload`。 +- 404/NoSuchUpload 作为幂等成功处理。 +- Abort 失败时保留原业务错误，并附加 cleanup 错误。 +- 析构函数不发起网络 Abort，只记录 warning 和 unfinished multipart metric。 + +### 5.6 DeleteObjects 与列举 + +Directory Bucket： + +- `DeleteObjects` 请求显式使用 CRC32C。 +- 批量删除响应中的所有 object error 都进入错误消息，不再只报告第一项。 +- 非 `/` 结尾的 list prefix 扩大到父目录 prefix。 +- 使用 continuation token 读取全部页面，再按原始 prefix 本地过滤。 +- BE 完整 list 结果在客户端按 key 排序。 +- 递归删除一旦删除了当前 batch，就丢弃旧 continuation token 并从头扫描，避免删除过程中继续使用失效的 opaque token。 + +### 5.7 BE 可观测性 + +新增指标： + +- CRC32C 校验失败数。 +- multipart Abort 次数和失败数。 +- writer 析构时仍存在 multipart upload 的次数。 +- Directory Bucket list page、扫描 key 和返回 key 数。 + +client 创建日志增加 bucket type、endpoint mode 和 checksum policy；credentials token 和 external ID 保持遮蔽。 + +## 6. Hive BE 上传、FE Complete 链路 + +Hive 写入不是在同一进程内完成 multipart：BE 上传 part，FE 在事务提交时 Complete。因此本次修改了 Thrift 协议。 + +修改文件： + +- `gensrc/thrift/DataSinks.thrift` +- `be/src/exec/sink/writer/vhive_partition_writer.cpp` +- `fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java` +- `fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java` +- `fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java` + +新增可选字段： + +```thrift +enum TObjectStorageChecksumAlgorithm { + CRC32C = 0 +} + +5: optional TObjectStorageChecksumAlgorithm checksum_algorithm +6: optional map part_checksums +``` + +传播链路： + +```text +BE S3FileWriter.completed_parts + | + v +VHivePartitionWriter + | + v +TS3MPUPendingUpload + etags + algorithm + part_checksums + | + v +FE HMSTransaction + | + v +ObjFileSystem -> UploadPartResult + | + v +S3ObjStorage CompleteMultipartUpload +``` + +Directory Bucket Complete 缺少任一 part checksum 时会失败，并进入 Abort。普通 S3 和旧消息仍可不设置新字段。 + +这些字段是 optional，因此协议可以解析旧消息，但混合版本集群仍不能安全启用 Directory Bucket：旧 BE 不产生 checksum，旧 FE 也不会使用 checksum。 + +## 7. FE 原生 S3 FileSystem 改动 + +### 7.1 S3ObjStorage 客户端模型 + +修改文件： + +- `fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java` + +原来的单一 lazy client 改为： + +- `generalClient` +- `directoryClient` +- 两者共享同一个长期 `AwsCredentialsProvider` + +每个公开对象操作解析当前 URI 的 bucket，并通过 `clientFor(bucket)` 选择 client。bucket type 不会被第一次请求永久缓存。 + +general client： + +- 保留 endpoint override、path-style、signer 和兼容配置。 +- 自定义 endpoint 禁用 Express session auth。 + +directory client： + +- 不设置 endpoint override。 +- 强制 virtual-hosted addressing。 +- 启用 SDK Express session auth。 +- 复用原 credentials provider。 + +Java Directory Bucket 请求增加： + +- PutObject CRC32C。 +- CreateMultipartUpload CRC32C。 +- UploadPart CRC32C，并要求响应包含 `ChecksumCRC32C`。 +- CompleteMultipartUpload 传播每个 part 的 CRC32C。 +- DeleteObjects CRC32C。 +- Abort 404 幂等处理。 + +Java SDK 负责根据 request body 计算请求 checksum；当前 Java 代码要求 UploadPart 响应 checksum 存在并继续传给 Complete，但没有像 C++ 路径一样单独计算本地 CRC32C 后做值比较。 + +### 7.2 Directory Bucket glob + +修改文件： + +- `fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java` +- `fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java` + +Directory Bucket glob 算法： + +1. 原始非 glob prefix 截断到最后一个 `/`。 +2. 请求中不发送 `StartAfter`。 +3. 用 opaque continuation token 读取所有服务页面。 +4. 本地应用原始 prefix、glob 和 `key > startAfter` 过滤。 +5. 使用 Doris UTF-8 binary comparator 排序。 +6. 排序后应用 `maxFiles`、`maxBytes` 和 `maxFile` 契约。 + +递归删除在 Directory Bucket 上删除一个 batch 后重新从空 token 列举。 + +当前实现为保证语义正确，会收集全部匹配对象后排序；尚未实现仅有 `maxFiles` 时的有界堆优化，也没有新增 Directory Bucket 专用的内存限制和查询取消检查。 + +### 7.3 S3FileSystemProperties + +修改文件： + +- `fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java` + +增加 Directory Bucket region、endpoint、HTTPS 和 path-style 校验。将 Directory Bucket 属性转换为 Hadoop S3A 配置时直接拒绝，防止进入未经验证的 S3A client。 + +## 8. fe-core 改动 + +### 8.1 S3Util + +`fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java` 的多个 builder 收敛到统一内部实现： + +- 已知 bucket 时先解析 capability。 +- Directory Bucket 不设置 endpoint override，不强制 `AwsS3V4Signer`，启用 Express session auth。 +- 普通 S3 保留原 builder 行为。 +- 自定义 S3-compatible endpoint 禁用 Express session auth。 +- HeadBucket connectivity tester 传入完整 bucket。 + +### 8.2 S3Resource + +`fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java` 增加： + +- Directory Bucket endpoint 缺少 scheme 时默认补 HTTPS，而不是 HTTP。 +- 在网络请求前校验 region、endpoint、path-style 和 HTTPS。 +- ping list 使用父目录扫描，并翻页直到确认测试对象存在。 +- multipart Complete 失败时执行 Abort。 +- Abort 失败作为 suppressed exception 保留在原始错误中。 + +### 8.3 S3URI + +`S3URI.isS3DirectoryBucket` 改为复用统一的 Directory Bucket 名称校验，不再维护一套宽松的 suffix 分割规则。 + +## 9. 不支持使用面的 fail-fast + +### 9.1 Cloud Storage Vault + +修改文件： + +- `fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java` +- `cloud/src/meta-service/meta_service_resource.cpp` + +FE 在生成 create/alter vault 请求前拒绝 Directory Bucket。Meta Service 在直接 ADD/ALTER object info 或 Storage Vault 时再次校验，并在加密或持久化前返回 `INVALID_ARGUMENT`。 + +门禁同时要求： + +- provider 为 S3。 +- endpoint 是官方 AWS service。 +- bucket 是合法 Directory Bucket 名称。 + +因此第三方 S3-compatible 服务中的 `--x-s3` bucket 不会被误拒绝。 + +### 9.2 Hadoop/S3A 与外部扫描 + +以下入口在第一个 Hadoop/S3A 网络请求前拒绝 Directory Bucket： + +| 使用面 | 拒绝位置 | +| --- | --- | +| 通用 Hadoop FileSystem | `DFSFileSystem.getHadoopFs` | +| 新 Hive connector | `HiveScanPlanProvider` 创建 Hadoop FileSystem 前 | +| Hudi | `HudiScanNode` 创建 Hudi client 前 | +| LakeSoul | `LakeSoulScanNode` 初始化 scanner 前 | +| Paimon | 各 Paimon metastore `initializeCatalog` 前 | +| Avro S3 reader | `S3FileReader` 调用 `FileSystem.get` 前 | + +上述门禁都会结合 endpoint 判断官方 AWS 与自定义兼容服务，普通 S3 和自定义 endpoint 保持原路径。 + +### 9.3 Presign 和 Copy + +- Java `S3ObjStorage.getPresignedUrl` 对 Directory Bucket 返回不支持。 +- BE AI embed 入口明确返回 `NotSupported`。 +- BE 通用 `generate_presigned_url` 接口受限于返回类型，只能记录 warning 并返回空字符串。 +- Java `S3ObjStorage.copyObject` 对 Directory Bucket 返回不支持。 + +当前 BE `S3ObjStorageClient::copy_object` 尚未增加 Directory Bucket 显式门禁；Directory Bucket Access Point 也没有专门的资源标识识别。这两项不能写入 P0 支持声明，需要在合入前补齐或证明没有用户可达调用链。 + +## 10. 普通 S3 与兼容性 + +本次实现有意保留以下行为： + +- 普通 AWS S3 继续使用 Content-MD5。 +- 普通 S3 client 继续使用原 endpoint override 和 path-style 配置。 +- MinIO、COS、OSS 等自定义 endpoint 不根据 bucket suffix 启用 Express。 +- `UploadPartResult` 保留原两参数构造，OSS、COS、OBS、Azure 不需要 checksum 字段。 +- `ObjStorageClient.abort_multipart_upload` 默认是 no-op，避免要求所有已有 object storage 实现同时增加原生 Abort。 +- bucket capability 不增加 PB、EditLog 或持久化字段。 +- 只有 Hive multipart 的 Thrift 消息增加 optional checksum 字段。 + +## 11. 配置示例 + +```sql +CREATE RESOURCE "s3_express_hot" +PROPERTIES +( + "type" = "s3", + "provider" = "S3", + "AWS_ENDPOINT" = "https://s3.us-east-1.amazonaws.com", + "AWS_REGION" = "us-east-1", + "AWS_BUCKET" = "analytics-hot--use1-az4--x-s3", + "AWS_ROOT_PATH" = "hot-data/", + "AWS_ACCESS_KEY" = "", + "AWS_SECRET_KEY" = "", + "use_path_style" = "false" +); +``` + +关键约束： + +- bucket 必须使用完整 Directory Bucket 名称。 +- endpoint 推荐配置标准 regional HTTPS endpoint，不需要手工配置 bucket-specific Zonal endpoint。 +- region 必须与 bucket 所在 region 一致。 +- `use_path_style` 必须为 `false`。 +- IAM principal 必须拥有目标 bucket 的 `s3express:CreateSession` 权限。 +- Doris 只配置长期 credentials provider，不能配置或持久化 SDK 产生的 Express session token。 + +## 12. 已增加但未执行的测试 + +### 12.1 C++ + +- `be/test/util/s3_util_test.cpp` + - 官方 AWS Directory Bucket 分类。 + - 自定义 endpoint 负例。 + - endpoint mode、region 和 zone 解析。 +- `be/test/io/fs/s3_file_writer_test.cpp` + - Complete 失败后执行 Abort。 + +### 12.2 Java + +- `S3BucketCapabilitiesTest` + - 分类、region、zone、dualstack 和 custom endpoint。 +- `S3ObjStorageMockTest` + - Put、Create MPU、UploadPart、Complete 和 DeleteObjects CRC32C request model。 + - Directory Bucket list 不发送 StartAfter。 +- `S3FileSystemTest` + - 无序 list 的本地排序和 `maxFile`。 +- `S3FileSystemPropertiesTest` + - Directory Bucket 拒绝转换为 S3A。 +- `DFSFileSystemTest`、`HiveScanPlanProviderTest`、`S3UtilsTest` + - S3A/Avro fail-fast 和第三方 endpoint 兼容性。 + +这些测试文件已经进入提交，但遵照开发要求没有执行。 + +## 13. 尚未完成的验证和风险 + +以下项目在代码提交后仍然是明确的未完成项： + +1. 没有执行 BE、FE 或 Cloud 编译，接口签名和依赖闭包尚未由编译器验证。 +2. 没有运行任何单元测试、回归测试或静态分析。 +3. C++ clang-format 未完成；当前环境缺少 Homebrew/clang-format。 +4. 没有真实 AWS Directory Bucket 测试： + - CreateSession 权限和 403 错误。 + - 连续运行超过五分钟的 session refresh。 + - 并发首次请求和 refresh stampede。 + - Put、multipart、DeleteObjects 的实际 HTTP checksum header。 + - 无序、多页 ListObjectsV2。 +5. Java UploadPart 没有单独计算本地 CRC32C 并和响应值比较，目前依赖 SDK request checksum，并要求响应 checksum 存在。 +6. Directory glob 尚未增加专用内存上限、取消检查和 `maxFiles` 有界堆优化。 +7. 跨语言分类测试尚未使用同一个测试向量文件。 +8. BE CopyObject 和 Directory Bucket Access Point 缺少明确门禁。 +9. Cloud Vault 门禁缺少绕过 FE 直调 Meta Service 的自动化测试。 +10. 当前未增加 cluster capability bit，混合版本启用限制依靠发布文档和运维流程。 + +因此，在上述验证完成前，只能表述为“Directory Bucket 支持代码已实现”，不能表述为“功能已经通过验收”或“可以作为 GA 发布”。 + +## 14. 建议验证顺序 + +后续验证建议按以下顺序进行： + +1. 运行 C++ 和 Java 格式检查，先解决纯工程问题。 +2. 编译 Thrift、BE、FE 和 Cloud，确认跨模块接口完整。 +3. 运行 capability、S3 object client、file writer、filesystem 和 fail-fast 单测。 +4. 运行普通 AWS S3、MinIO/S3-compatible 回归，确认兼容路径没有变化。 +5. 使用真实 AWS Directory Bucket 验证小对象、multipart、Abort、DeleteObjects 和多页 list。 +6. 让 client 跨越至少一次五分钟 session 过期，并执行并发刷新验证。 +7. 验证 Cloud Meta Service 直接 add/alter vault 均在持久化前拒绝。 +8. 补齐 BE CopyObject、Access Point 等剩余边界后，再评估 P0 验收。 + +## 15. 回滚与混合版本 + +- 本次不改变对象 key 和对象内容格式。 +- 旧 Doris binary 不能可靠访问仍位于 Directory Bucket 中的数据，因此不能把二进制回滚等同于数据可用性回滚。 +- 回滚前必须停止 Directory Bucket workload，把仍需访问的数据复制到普通 S3，并切换 Resource。 +- 混合版本期间只能继续使用普通 S3；所有可能执行该资源 I/O 的 BE 和 FE 升级完成后，才能启用 Directory Bucket Resource。 +- Cloud Storage Vault 当前主动拒绝 Directory Bucket，不存在透明切换或回滚路径。 + +## 16. 结论 + +本次改动已经完成 S3 Express One Zone 的核心代码闭环：统一能力识别、SDK-managed session auth、CRC32C、multipart checksum 传播、失败 Abort、Directory Bucket list 语义、FE 原生 S3 client，以及未支持 surface 的 fail-fast。 + +当前最重要的下一步不是继续扩大支持面，而是完成编译、普通 S3 兼容回归和真实 Directory Bucket 集成验证，并补齐本文列出的 BE CopyObject、Access Point、资源限制和 Cloud 直调测试缺口。 From 1c134379a31684309a1fae956032de5c1f89161f Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 17 Jul 2026 16:16:21 +0800 Subject: [PATCH 07/13] [feature](s3) Support native S3 Express object operations ### What problem does this PR solve? Issue Number: None Related PR: #63409 Problem Summary: The previous S3 Express implementation expanded Directory Bucket awareness across listing, deletion, presigning, connectors, Cloud storage, checksums, and documentation. This change narrows support to official AWS endpoints and valid --x-s3 bucket names on Doris native known-object paths. AWS SDK endpoint rules provide the zonal endpoint and CreateSession authentication, Express clients use HTTPS and virtual-hosted addressing, Head/Get/range/Put and multipart create/upload/complete/abort are supported, exact S3 TVF and S3 Load paths use HeadObject, and Express uploads no longer send Content-MD5. General-purpose S3 and S3-compatible endpoints retain their existing client behavior. ### Release note Support native known-object reads and writes for Amazon S3 Express One Zone Directory Buckets. ### Check List (For Author) - Test: Not run (per user request) - Behavior changed: Yes. Official AWS Directory Buckets use SDK S3 Express routing and session authentication for native known-object operations. - Does this need documentation: Yes. The user documentation will be handled separately. --- .../sink/writer/vhive_partition_writer.cpp | 11 - be/src/exprs/function/ai/embed.h | 15 +- be/src/io/fs/obj_storage_client.h | 2 - be/src/io/fs/s3_file_writer.cpp | 28 +- be/src/io/fs/s3_obj_storage_client.cpp | 152 +- be/src/io/fs/s3_obj_storage_client.h | 6 +- be/src/io/tools/file_cache_microbench.cpp | 1 - be/src/util/s3_util.cpp | 150 +- be/src/util/s3_util.h | 57 +- .../io/fs/s3_obj_stroage_client_mock_test.cpp | 39 +- be/test/util/s3_util_test.cpp | 45 +- .../meta-service/meta_service_resource.cpp | 27 - common/cpp/aws_logger.h | 9 - common/cpp/s3_bucket_capabilities.h | 200 --- .../org/apache/doris/avro/S3FileReader.java | 4 - .../java/org/apache/doris/avro/S3Utils.java | 30 - .../org/apache/doris/avro/S3UtilsTest.java | 36 - fe/fe-connector/fe-connector-hive/pom.xml | 6 - .../connector/hive/HiveScanPlanProvider.java | 13 - .../hive/HiveScanPlanProviderTest.java | 43 - .../org/apache/doris/catalog/S3Resource.java | 39 +- .../apache/doris/catalog/StorageVaultMgr.java | 7 - .../org/apache/doris/common/util/S3URI.java | 27 +- .../org/apache/doris/common/util/S3Util.java | 150 +- ...bstractS3CompatibleConnectivityTester.java | 2 +- .../doris/datasource/hive/HMSTransaction.java | 12 +- .../datasource/hudi/source/HudiScanNode.java | 12 +- .../lakesoul/source/LakeSoulScanNode.java | 9 - .../metastore/AbstractPaimonProperties.java | 15 - .../PaimonAliyunDLFMetaStoreProperties.java | 1 - .../PaimonFileSystemMetaStoreProperties.java | 1 - .../PaimonHMSMetaStoreProperties.java | 1 - .../PaimonJdbcMetaStoreProperties.java | 1 - .../PaimonRestMetaStoreProperties.java | 1 - .../load/loadv2/BrokerLoadPendingTask.java | 24 +- .../ExternalFileTableValuedFunction.java | 24 +- .../apache/doris/common/util/S3UtilTest.java | 14 +- .../filesystem/S3BucketCapabilities.java | 277 ---- .../doris/filesystem/UploadPartResult.java | 10 - .../filesystem/S3BucketCapabilitiesTest.java | 93 -- .../doris/filesystem/hdfs/DFSFileSystem.java | 8 - .../filesystem/hdfs/DFSFileSystemTest.java | 23 - .../doris/filesystem/s3/S3FileSystem.java | 88 +- .../filesystem/s3/S3FileSystemProperties.java | 12 +- .../doris/filesystem/s3/S3ObjStorage.java | 224 ++- .../s3/S3FileSystemPropertiesTest.java | 14 - .../doris/filesystem/s3/S3FileSystemTest.java | 25 +- .../filesystem/s3/S3ObjStorageMockTest.java | 87 -- .../doris/filesystem/spi/ObjFileSystem.java | 12 +- .../spi/S3CompatibleFileSystem.java | 19 +- gensrc/thrift/DataSinks.thrift | 6 - goal.md | 1225 ----------------- s3_express_one_zone_implementation.md | 534 ------- thirdparty/vars.sh | 8 +- 54 files changed, 499 insertions(+), 3380 deletions(-) delete mode 100644 common/cpp/s3_bucket_capabilities.h delete mode 100644 fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java delete mode 100644 fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java delete mode 100644 fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java delete mode 100644 fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java delete mode 100644 goal.md delete mode 100644 s3_express_one_zone_implementation.md diff --git a/be/src/exec/sink/writer/vhive_partition_writer.cpp b/be/src/exec/sink/writer/vhive_partition_writer.cpp index 2f4bf8f920b156..8331efac54bd47 100644 --- a/be/src/exec/sink/writer/vhive_partition_writer.cpp +++ b/be/src/exec/sink/writer/vhive_partition_writer.cpp @@ -19,7 +19,6 @@ #include -#include "common/check.h" #include "core/block/materialize_block.h" #include "core/column/column_map.h" #include "format/transformer/vcsv_transformer.h" @@ -198,20 +197,10 @@ bool VHivePartitionWriter::_build_s3_mpu_pending_upload(TS3MPUPendingUpload* pen pending_upload->__set_upload_id(upload_id); std::map etags; - std::map part_checksums; for (auto& completed_part : s3_mpu_file_writer->completed_parts()) { etags.insert({completed_part.part_num, completed_part.etag}); - if (completed_part.checksum_crc32c.has_value()) { - part_checksums.insert( - {completed_part.part_num, *completed_part.checksum_crc32c}); - } } pending_upload->__set_etags(etags); - if (!part_checksums.empty()) { - DORIS_CHECK_EQ(part_checksums.size(), etags.size()); - pending_upload->__set_checksum_algorithm(TObjectStorageChecksumAlgorithm::CRC32C); - pending_upload->__set_part_checksums(part_checksums); - } return true; } diff --git a/be/src/exprs/function/ai/embed.h b/be/src/exprs/function/ai/embed.h index 74758c251df08a..2367e4b9459540 100644 --- a/be/src/exprs/function/ai/embed.h +++ b/be/src/exprs/function/ai/embed.h @@ -374,21 +374,16 @@ class FunctionEmbed : public AIFunction { S3ClientConf s3_client_conf; RETURN_IF_ERROR(init_s3_client_conf_from_json(file_input, s3_client_conf)); + auto s3_client = S3ClientFactory::instance().create(s3_client_conf); + if (s3_client == nullptr) { + return Status::InternalError("Failed to create S3 client for EMBED file input"); + } + S3URI s3_uri(uri); RETURN_IF_ERROR(s3_uri.parse()); std::string bucket = s3_uri.get_bucket(); std::string key = s3_uri.get_key(); DORIS_CHECK(!bucket.empty() && !key.empty()); - s3_client_conf.bucket = bucket; - if (resolve_s3_bucket_capabilities(bucket, s3_client_conf.endpoint) - .is_directory_bucket()) { - return Status::NotSupported( - "Presigned URL is not supported for AWS Directory Bucket"); - } - auto s3_client = S3ClientFactory::instance().create(s3_client_conf); - if (s3_client == nullptr) { - return Status::InternalError("Failed to create S3 client for EMBED file input"); - } media_url = s3_client->generate_presigned_url({.bucket = bucket, .key = key}, ttl_seconds, s3_client_conf); return Status::OK(); diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 083d0e905b2b29..4a6b847ae30b10 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -50,7 +50,6 @@ struct ObjectStoragePathOptions { struct ObjectCompleteMultiPart { int part_num = 0; std::string etag = std::string(); - std::optional checksum_crc32c = std::nullopt; }; struct ObjectStorageStatus { @@ -77,7 +76,6 @@ struct ObjectStorageUploadResponse { ObjectStorageResponse resp {}; std::optional upload_id = std::nullopt; std::optional etag = std::nullopt; - std::optional checksum_crc32c = std::nullopt; }; struct ObjectStorageHeadResponse : ObjectStorageResponse { diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index 7f4dd80c9f39c4..c589759fd778bd 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -85,12 +85,8 @@ S3FileWriter::~S3FileWriter() { _wait_until_finish(fmt::format("wait s3 file {} upload to be finished", _obj_storage_path_opts.path.native())); } - if (_obj_storage_path_opts.upload_id.has_value() && state() == State::OPENED) { - s3_bvar::s3_multipart_unfinished_on_destroy_total << 1; - LOG(WARNING) << "S3FileWriter destroyed with an unfinished multipart upload, bucket=" - << _obj_storage_path_opts.bucket << ", key=" << _obj_storage_path_opts.key - << ", upload_id=" << *_obj_storage_path_opts.upload_id; - } + // Do not issue network requests from the destructor. Close failures abort an active + // multipart upload in _close_impl(). if (state() == State::OPENED && !_failed) { s3_bytes_written_total << _bytes_appended; } @@ -430,8 +426,7 @@ void S3FileWriter::_upload_one_part(int part_num, UploadFileBuffer& buf) { s3_bytes_written_total << buf.get_size(); ObjectCompleteMultiPart completed_part { - part_num, resp.etag.has_value() ? std::move(resp.etag.value()) : "", - std::move(resp.checksum_crc32c)}; + part_num, resp.etag.has_value() ? std::move(resp.etag.value()) : ""}; std::unique_lock lck {_completed_lock}; _completed_parts.emplace_back(std::move(completed_part)); @@ -504,6 +499,11 @@ Status S3FileWriter::_complete() { _wait_until_finish("Complete"); TEST_SYNC_POINT_CALLBACK("S3FileWriter::_complete:1", std::make_pair(&_failed, &_completed_parts)); + if (_used_by_s3_committer) { // S3 committer will complete multipart upload file on FE side. + s3_file_created_total << 1; // Assume that it will be created successfully + return Status::OK(); + } + // check number of parts int64_t expected_num_parts1 = (_bytes_appended / config::s3_write_buffer_size) + !!(_bytes_appended % config::s3_write_buffer_size); @@ -526,18 +526,6 @@ Status S3FileWriter::_complete() { // make sure _completed_parts are ascending order std::sort(_completed_parts.begin(), _completed_parts.end(), [](auto& p1, auto& p2) { return p1.part_num < p2.part_num; }); - for (size_t i = 0; i < _completed_parts.size(); ++i) { - if (_completed_parts[i].part_num != static_cast(i + 1)) { - return Status::InternalError( - "multipart upload parts must be consecutive from 1, completed_parts_list={} " - "file_path={}", - _dump_completed_part(), _obj_storage_path_opts.path.native()); - } - } - if (_used_by_s3_committer) { // S3 committer will complete multipart upload file on FE side. - s3_file_created_total << 1; // Assume that it will be created successfully - return Status::OK(); - } TEST_SYNC_POINT_CALLBACK("S3FileWriter::_complete:2", &_completed_parts); LOG(INFO) << "complete_multipart_upload " << _obj_storage_path_opts.path.native() << " size=" << _bytes_appended << " number_parts=" << _completed_parts.size() diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index ebed68001a701a..0792046d320f9b 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include #include @@ -61,12 +60,10 @@ #include #include #include -#include #include #include -#include "common/config.h" #include "common/logging.h" #include "common/status.h" #include "cpp/sync_point.h" @@ -119,47 +116,11 @@ using namespace Aws::S3::Model; static constexpr int S3_REQUEST_THRESHOLD_MS = 5000; -namespace { -// AWS expects the CRC32C value as the big-endian 4-byte representation, base64-encoded. -Aws::String compute_crc32c_b64(std::string_view data) { - uint32_t crc = crc32c::Crc32c(reinterpret_cast(data.data()), data.size()); - uint8_t bytes[4] = {static_cast((crc >> 24) & 0xff), - static_cast((crc >> 16) & 0xff), - static_cast((crc >> 8) & 0xff), static_cast(crc & 0xff)}; - return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::ByteBuffer(bytes, sizeof(bytes))); -} - -std::string directory_list_prefix(std::string_view prefix) { - if (prefix.empty() || prefix.ends_with('/')) { - return std::string(prefix); - } - const auto slash = prefix.rfind('/'); - return slash == std::string_view::npos ? "" : std::string(prefix.substr(0, slash + 1)); -} - -std::string delete_errors_message(const Aws::Vector& errors, - const Aws::String& request_id) { - std::string message = fmt::format("DeleteObjects partially failed, request_id={}", request_id); - for (const auto& error : errors) { - fmt::format_to(std::back_inserter(message), ", key={}, code={}, message={}", error.GetKey(), - error.GetCode(), error.GetMessage()); - } - return message; -} -} // namespace - -S3ObjStorageClient::S3ObjStorageClient(std::shared_ptr client, - S3BucketCapabilities capabilities) - : _client(std::move(client)), _capabilities(capabilities) {} - ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( const ObjectStoragePathOptions& opts) { CreateMultipartUploadRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); request.SetContentType("application/octet-stream"); - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); - } MonotonicStopWatch watch; watch.start(); @@ -197,11 +158,7 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti Aws::S3::Model::PutObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); auto string_view_stream = std::make_shared(stream.data(), stream.size()); - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - // S3 Express One Zone rejects Content-MD5; use CRC32C instead. - request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); - request.SetChecksumCRC32C(compute_crc32c_b64(stream)); - } else { + if (!_is_s3_express) { Aws::Utils::ByteBuffer part_md5( Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); @@ -248,13 +205,7 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP request.SetBody(string_view_stream); - std::optional checksum_crc32c; - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - // S3 Express One Zone rejects Content-MD5; use CRC32C instead. - checksum_crc32c = compute_crc32c_b64(stream); - request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); - request.SetChecksumCRC32C(*checksum_crc32c); - } else { + if (!_is_s3_express) { Aws::Utils::ByteBuffer part_md5( Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); @@ -295,23 +246,7 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP << "UploadPart cost=" << watch.elapsed_time_milliseconds() << "ms" << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key << ", part_num=" << part_num << ", upload_id=" << *opts.upload_id; - if (checksum_crc32c.has_value() && - outcome.GetResult().GetChecksumCRC32C() != *checksum_crc32c) { - s3_bvar::s3_checksum_failure_total << 1; - auto st = Status::IOError( - "CRC32C mismatch for UploadPart, bucket={}, key={}, part={}, expected={}, " - "actual={}, request_id={}", - opts.bucket, opts.key, part_num, *checksum_crc32c, - outcome.GetResult().GetChecksumCRC32C(), request_id); - return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response(std::move(st)), 200, request_id}}; - } - return ObjectStorageUploadResponse { - .etag = outcome.GetResult().GetETag(), - .checksum_crc32c = checksum_crc32c.has_value() - ? std::optional(std::string( - checksum_crc32c->c_str(), checksum_crc32c->size())) - : std::nullopt}; + return ObjectStorageUploadResponse {.etag = outcome.GetResult().GetETag()}; } ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( @@ -320,31 +255,13 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( CompleteMultipartUploadRequest request; request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - for (size_t i = 0; i < completed_parts.size(); ++i) { - const auto& part = completed_parts[i]; - if (part.part_num != static_cast(i + 1)) { - return {convert_to_obj_response(Status::InvalidArgument( - "Directory Bucket multipart parts must be consecutive from 1"))}; - } - if (!part.checksum_crc32c.has_value()) { - return {convert_to_obj_response(Status::InvalidArgument( - "AWS Directory Bucket multipart completion requires CRC32C for part {}", - part.part_num))}; - } - } - } - CompletedMultipartUpload completed_upload; std::vector complete_parts; std::ranges::transform(completed_parts, std::back_inserter(complete_parts), - [this](const ObjectCompleteMultiPart& part_ptr) { + [](const ObjectCompleteMultiPart& part_ptr) { CompletedPart part; part.SetPartNumber(part_ptr.part_num); part.SetETag(part_ptr.etag); - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - part.SetChecksumCRC32C(*part_ptr.checksum_crc32c); - } return part; }); completed_upload.SetParts(std::move(complete_parts)); @@ -385,12 +302,10 @@ ObjectStorageResponse S3ObjStorageClient::abort_multipart_upload( Aws::S3::Model::AbortMultipartUploadRequest request; request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); auto outcome = s3_put_rate_limit([&]() { return _client->AbortMultipartUpload(request); }); - s3_bvar::s3_multipart_abort_total << 1; if (outcome.IsSuccess() || outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { return ObjectStorageResponse::OK(); } - s3_bvar::s3_multipart_abort_failure_total << 1; return {convert_to_obj_response(s3fs_error( outcome.GetError(), fmt::format("failed to AbortMultipartUpload: {}, upload_id={}", @@ -451,10 +366,7 @@ ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOpti ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, std::vector* files) { Aws::S3::Model::ListObjectsV2Request request; - std::string service_prefix = _capabilities.is_directory_bucket() - ? directory_list_prefix(opts.prefix) - : opts.prefix; - request.WithBucket(opts.bucket).WithPrefix(service_prefix); + request.WithBucket(opts.bucket).WithPrefix(opts.prefix); bool is_trucated = false; do { Aws::S3::Model::ListObjectsV2Outcome outcome; @@ -478,25 +390,14 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp static_cast(outcome.GetError().GetResponseCode()), outcome.GetError().GetRequestId()}; } - if (_capabilities.is_directory_bucket()) { - s3_bvar::s3_directory_list_pages << 1; - s3_bvar::s3_directory_list_scanned_keys - << outcome.GetResult().GetContents().size(); - } for (const auto& obj : outcome.GetResult().GetContents()) { std::string key = obj.GetKey(); - if (!key.starts_with(opts.prefix)) { - continue; - } bool is_dir = (key.back() == '/'); FileInfo file_info; file_info.file_name = obj.GetKey(); file_info.file_size = obj.GetSize(); file_info.is_file = !is_dir; files->push_back(std::move(file_info)); - if (_capabilities.is_directory_bucket()) { - s3_bvar::s3_directory_list_returned_keys << 1; - } } is_trucated = outcome.GetResult().GetIsTruncated(); if (is_trucated && outcome.GetResult().GetNextContinuationToken().empty()) { @@ -508,9 +409,6 @@ ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOp request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); } while (is_trucated); - if (!_capabilities.list_is_lexicographic) { - std::ranges::sort(*files, {}, &FileInfo::file_name); - } return ObjectStorageResponse::OK(); } @@ -518,9 +416,6 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath std::vector objs) { Aws::S3::Model::DeleteObjectsRequest delete_request; delete_request.SetBucket(opts.bucket); - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - delete_request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); - } Aws::S3::Model::Delete del; Aws::Vector objects; std::ranges::transform(objs, std::back_inserter(objects), [](auto&& obj_key) { @@ -543,9 +438,10 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePath // case for partial delete object failure SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); if (!delete_outcome.GetResult().GetErrors().empty()) { - return {convert_to_obj_response(Status::InternalError(delete_errors_message( - delete_outcome.GetResult().GetErrors(), - delete_outcome.GetResult().GetRequestId())))}; + const auto& e = delete_outcome.GetResult().GetErrors().front(); + return {convert_to_obj_response( + Status::InternalError("failed to delete object {}: {}, request_id={}", e.GetKey(), + e.GetMessage(), delete_outcome.GetResult().GetRequestId()))}; } return ObjectStorageResponse::OK(); } @@ -569,15 +465,9 @@ ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathO ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( const ObjectStoragePathOptions& opts) { Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket) - .WithPrefix(_capabilities.is_directory_bucket() - ? directory_list_prefix(opts.prefix) - : opts.prefix); + request.WithBucket(opts.bucket).WithPrefix(opts.prefix); Aws::S3::Model::DeleteObjectsRequest delete_request; delete_request.SetBucket(opts.bucket); - if (_capabilities.checksum_policy == S3ChecksumPolicy::CRC32C) { - delete_request.SetChecksumAlgorithm(ChecksumAlgorithm::CRC32C); - } bool is_trucated = false; do { Aws::S3::Model::ListObjectsV2Outcome outcome; @@ -596,11 +486,8 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( Aws::Vector objects; objects.reserve(result.GetContents().size()); for (const auto& obj : result.GetContents()) { - if (obj.GetKey().starts_with(opts.prefix)) { - objects.emplace_back().SetKey(obj.GetKey()); - } + objects.emplace_back().SetKey(obj.GetKey()); } - bool deleted_objects = false; if (!objects.empty()) { Aws::S3::Model::Delete del; del.WithObjects(std::move(objects)).SetQuiet(true); @@ -619,16 +506,14 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", &delete_outcome); if (!delete_outcome.GetResult().GetErrors().empty()) { - return {convert_to_obj_response(Status::InternalError(delete_errors_message( - delete_outcome.GetResult().GetErrors(), - delete_outcome.GetResult().GetRequestId())))}; + const auto& e = delete_outcome.GetResult().GetErrors().front(); + return {convert_to_obj_response(Status::InternalError( + "failed to delete object {}: {}, request_id={}", opts.key, e.GetMessage(), + delete_outcome.GetResult().GetRequestId()))}; } - deleted_objects = true; } is_trucated = result.GetIsTruncated(); - request.SetContinuationToken(_capabilities.is_directory_bucket() && deleted_objects - ? Aws::String() - : result.GetNextContinuationToken()); + request.SetContinuationToken(result.GetNextContinuationToken()); } while (is_trucated); return ObjectStorageResponse::OK(); } @@ -636,11 +521,6 @@ ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, int64_t expiration_secs, const S3ClientConf&) { - if (!_capabilities.supports_presign) { - LOG(WARNING) << "Presigned URL is not supported for AWS Directory Bucket, bucket=" - << opts.bucket; - return {}; - } return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, expiration_secs); } diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 27ce8686dd1e5c..70088934afea25 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -17,7 +17,6 @@ #pragma once -#include "cpp/s3_bucket_capabilities.h" #include "io/fs/obj_storage_client.h" #include "io/fs/s3_file_system.h" @@ -34,7 +33,8 @@ class ObjClientHolder; class S3ObjStorageClient final : public ObjStorageClient { public: explicit S3ObjStorageClient(std::shared_ptr client, - S3BucketCapabilities capabilities = {}); + bool is_s3_express = false) + : _client(std::move(client)), _is_s3_express(is_s3_express) {} ~S3ObjStorageClient() override = default; ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override; @@ -61,7 +61,7 @@ class S3ObjStorageClient final : public ObjStorageClient { private: std::shared_ptr _client; - S3BucketCapabilities _capabilities; + bool _is_s3_express = false; }; } // namespace doris::io diff --git a/be/src/io/tools/file_cache_microbench.cpp b/be/src/io/tools/file_cache_microbench.cpp index ccc8e60f3899b2..46dfb6934fd928 100644 --- a/be/src/io/tools/file_cache_microbench.cpp +++ b/be/src/io/tools/file_cache_microbench.cpp @@ -1495,7 +1495,6 @@ class JobManager { s3_conf.sk = doris::config::test_s3_sk; s3_conf.region = doris::config::test_s3_region; s3_conf.endpoint = doris::config::test_s3_endpoint; - s3_conf.bucket = doris::config::test_s3_bucket; return s3_conf; } diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index af869772f94766..a484188ed89d73 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -28,11 +28,14 @@ #include #include #include +#include #include #include #include +#include #include +#include #include "util/string_util.h" @@ -78,62 +81,60 @@ bvar::LatencyRecorder s3_list_latency("s3_list"); bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); -bvar::Adder s3_checksum_failure_total("s3_checksum_failure_total"); -bvar::Adder s3_multipart_abort_total("s3_multipart_abort_total"); -bvar::Adder s3_multipart_abort_failure_total("s3_multipart_abort_failure_total"); -bvar::Adder s3_multipart_unfinished_on_destroy_total( - "s3_multipart_unfinished_on_destroy_total"); -bvar::Adder s3_directory_list_scanned_keys("s3_directory_list_scanned_keys"); -bvar::Adder s3_directory_list_returned_keys("s3_directory_list_returned_keys"); -bvar::Adder s3_directory_list_pages("s3_directory_list_pages"); }; // namespace s3_bvar namespace { +std::string s3_endpoint_host(std::string_view endpoint) { + const auto scheme = endpoint.find("://"); + if (scheme != std::string_view::npos) { + endpoint.remove_prefix(scheme + 3); + } + endpoint = endpoint.substr(0, endpoint.find_first_of("/?#")); + if (const auto port = endpoint.find(':'); port != std::string_view::npos) { + endpoint = endpoint.substr(0, port); + } + std::string host(endpoint); + std::transform(host.begin(), host.end(), host.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return host; +} + +bool is_official_aws_s3_endpoint(std::string_view endpoint) { + const auto host = s3_endpoint_host(endpoint); + const bool aws_dns = host.ends_with(".amazonaws.com") || + host.ends_with(".amazonaws.com.cn") || host.ends_with(".api.aws"); + return aws_dns && + (host.starts_with("s3.") || host.starts_with("s3-") || + host.starts_with("s3express-") || host.find(".s3.") != std::string::npos || + host.find(".s3-") != std::string::npos || + host.find(".s3express-") != std::string::npos); +} + +bool is_s3_directory_bucket_name(std::string_view bucket) { + constexpr std::string_view suffix = "--x-s3"; + if (!bucket.ends_with(suffix)) { + return false; + } + bucket.remove_suffix(suffix.size()); + const auto zone_separator = bucket.rfind("--"); + if (zone_separator == std::string_view::npos || zone_separator == 0) { + return false; + } + const auto zone_id = bucket.substr(zone_separator + 2); + return zone_id.find("-az") != std::string_view::npos; +} + doris::Status is_s3_conf_valid(const S3ClientConf& conf) { - if (conf.endpoint.empty() && conf.provider == io::ObjStorageType::AZURE) { + if (conf.endpoint.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty endpoint"); } if (conf.region.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } - - const auto capabilities = resolve_s3_bucket_capabilities(conf.bucket, conf.endpoint); - if (capabilities.official_aws_service && is_s3express_control_endpoint(conf.endpoint)) { - return Status::InvalidArgument( - "s3express-control endpoint cannot be used for object operations"); - } - if (capabilities.official_aws_service && is_s3express_zonal_endpoint(conf.endpoint) && - !capabilities.is_directory_bucket()) { + if (is_s3_express(conf.bucket, conf.endpoint) && !conf.use_virtual_addressing) { return Status::InvalidArgument( - "An S3 Express endpoint requires a valid AWS Directory Bucket name"); - } - if (capabilities.is_directory_bucket()) { - if (capabilities.endpoint_mode != S3EndpointMode::AWS_SDK_RULES) { - return Status::InvalidArgument( - "AWS Directory Bucket requires a standard regional or zonal S3 endpoint"); - } - if (!conf.use_virtual_addressing) { - return Status::InvalidArgument( - "Path-style addressing is not supported for AWS Directory Bucket"); - } - if (config::s3_client_http_scheme == "http" || conf.endpoint.starts_with("http://")) { - return Status::InvalidArgument("AWS Directory Bucket requires HTTPS"); - } - const auto endpoint_region = aws_s3_endpoint_region(conf.endpoint); - if (!endpoint_region.empty() && endpoint_region != conf.region) { - return Status::InvalidArgument( - "Configured endpoint region {} does not match AWS_REGION {} for Directory " - "Bucket {}", - endpoint_region, conf.region, conf.bucket); - } - const auto endpoint_zone = s3express_endpoint_zone_id(conf.endpoint); - const auto bucket_zone = s3_directory_bucket_zone_id(conf.bucket); - if (!endpoint_zone.empty() && endpoint_zone != bucket_zone) { - return Status::InvalidArgument( - "Configured endpoint zone {} does not match Directory Bucket zone {}", - endpoint_zone, bucket_zone); - } + "Path-style addressing is not supported for AWS Directory Bucket"); } if (conf.role_arn.empty()) { @@ -534,24 +535,15 @@ std::shared_ptr S3ClientFactory::get_aws_cred std::shared_ptr S3ClientFactory::_create_s3_client( const S3ClientConf& s3_conf) { + const bool s3_express = is_s3_express(s3_conf.bucket, s3_conf.endpoint); TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", std::make_shared(std::make_shared())); - // Use S3ClientConfiguration (not the legacy ClientConfiguration) so the SDK's - // endpoint-rules resolver is active. This is required for S3 Express One Zone: - // the resolver detects the --x-s3 bucket suffix, calls CreateSession, and signs - // requests with service name "s3express" instead of "s3". - const auto capabilities = - resolve_s3_bucket_capabilities(s3_conf.bucket, s3_conf.endpoint); - const auto payload_signing_policy = - capabilities.is_directory_bucket() - ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent - : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; - Aws::S3::S3ClientConfiguration aws_config(S3ClientFactory::getClientConfiguration(), - payload_signing_policy, - s3_conf.use_virtual_addressing); + Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); + if (s3_conf.need_override_endpoint && !s3_express) { + aws_config.endpointOverride = s3_conf.endpoint; + } aws_config.region = s3_conf.region; - aws_config.useVirtualAddressing = s3_conf.use_virtual_addressing; if (_ca_cert_file_path.empty()) { _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); @@ -567,6 +559,7 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.maxConnections = 102400; } + aws_config.requestTimeoutMs = 30000; if (s3_conf.request_timeout_ms > 0) { aws_config.requestTimeoutMs = s3_conf.request_timeout_ms; } @@ -575,33 +568,34 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms; } - if (config::s3_client_http_scheme == "http") { + if (s3_express) { + aws_config.scheme = Aws::Http::Scheme::HTTPS; + } else if (config::s3_client_http_scheme == "http") { aws_config.scheme = Aws::Http::Scheme::HTTP; } aws_config.retryStrategy = std::make_shared( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/false); - // Directory Bucket requests must remain on the SDK endpoint-rules path so it can resolve - // the bucket-specific Zonal host and manage CreateSession credentials. - if (s3_conf.need_override_endpoint && !s3_conf.endpoint.empty() && - !capabilities.is_directory_bucket()) { - aws_config.endpointOverride = s3_conf.endpoint; + std::shared_ptr new_client; + if (s3_express) { + Aws::S3::S3ClientConfiguration express_config( + aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent, + true); + express_config.disableS3ExpressAuth = false; + new_client = std::make_shared( + get_aws_credentials_provider(s3_conf), + Aws::MakeShared("S3Client"), express_config); + } else { + new_client = std::make_shared( + get_aws_credentials_provider(s3_conf), std::move(aws_config), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + s3_conf.use_virtual_addressing); } - aws_config.disableS3ExpressAuth = !capabilities.official_aws_service; - - auto new_client = std::make_shared( - get_aws_credentials_provider(s3_conf), - Aws::MakeShared("S3Client"), aws_config); auto obj_client = - std::make_shared(std::move(new_client), capabilities); - LOG_INFO("create one s3 client with {}, bucket_type={}, endpoint_mode={}, checksum_policy={}", - s3_conf.to_string(), - capabilities.is_directory_bucket() ? "directory" : "general_purpose", - capabilities.endpoint_mode == S3EndpointMode::AWS_SDK_RULES ? "sdk_rules" - : "override", - capabilities.checksum_policy == S3ChecksumPolicy::CRC32C ? "crc32c" : "content_md5"); + std::make_shared(std::move(new_client), s3_express); + LOG_INFO("create one s3 client with {}", s3_conf.to_string()); return obj_client; } @@ -863,7 +857,7 @@ std::string hide_access_key(const std::string& ak) { } bool is_s3_express(std::string_view bucket, std::string_view endpoint) { - return resolve_s3_bucket_capabilities(bucket, endpoint).is_directory_bucket(); + return is_s3_directory_bucket_name(bucket) && is_official_aws_s3_endpoint(endpoint); } } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 5a849b61734e3f..e73dc3a9dbcae4 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -30,12 +30,12 @@ #include #include #include +#include #include #include "common/status.h" #include "core/string_ref.h" #include "cpp/aws_common.h" -#include "cpp/s3_bucket_capabilities.h" #include "cpp/token_bucket_rate_limiter.h" #include "io/fs/obj_storage_client.h" @@ -61,13 +61,6 @@ extern bvar::LatencyRecorder s3_list_latency; extern bvar::LatencyRecorder s3_list_object_versions_latency; extern bvar::LatencyRecorder s3_get_bucket_version_latency; extern bvar::LatencyRecorder s3_copy_object_latency; -extern bvar::Adder s3_checksum_failure_total; -extern bvar::Adder s3_multipart_abort_total; -extern bvar::Adder s3_multipart_abort_failure_total; -extern bvar::Adder s3_multipart_unfinished_on_destroy_total; -extern bvar::Adder s3_directory_list_scanned_keys; -extern bvar::Adder s3_directory_list_returned_keys; -extern bvar::Adder s3_directory_list_pages; }; // namespace s3_bvar std::string hide_access_key(const std::string& ak); @@ -100,29 +93,21 @@ struct S3ClientConf { uint64_t get_hash() const { uint64_t hash_code = 0; - auto combine = [&hash_code](uint64_t value) { - hash_code ^= value + 0x9e3779b97f4a7c15ULL + (hash_code << 6) + (hash_code >> 2); - }; - combine(crc32_hash(ak)); - combine(crc32_hash(sk)); - combine(crc32_hash(token)); - combine(crc32_hash(endpoint)); - combine(crc32_hash(region)); - combine(crc32_hash(bucket)); - combine(max_connections); - combine(request_timeout_ms); - combine(connect_timeout_ms); - combine(use_virtual_addressing); - combine(need_override_endpoint); - combine(static_cast(provider)); - combine(static_cast(cred_provider_type)); - combine(crc32_hash(role_arn)); - combine(crc32_hash(external_id)); - const auto capabilities = resolve_s3_bucket_capabilities(bucket, endpoint); - combine(static_cast(capabilities.bucket_type)); - combine(static_cast(capabilities.endpoint_mode)); - combine(static_cast(capabilities.checksum_policy)); - combine(crc32_hash(config::s3_client_http_scheme)); + // Use crc32_hash(ak + sk) hash to prevent swapped AK/SK order from producing same result. + hash_code ^= crc32_hash(ak + sk); + hash_code ^= crc32_hash(token); + hash_code ^= crc32_hash(endpoint); + hash_code ^= crc32_hash(region); + hash_code ^= crc32_hash(bucket); + hash_code ^= max_connections; + hash_code ^= request_timeout_ms; + hash_code ^= connect_timeout_ms; + hash_code ^= use_virtual_addressing; + hash_code ^= static_cast(provider); + + hash_code ^= static_cast(cred_provider_type); + hash_code ^= crc32_hash(role_arn); + hash_code ^= crc32_hash(external_id); return hash_code; } @@ -130,12 +115,10 @@ struct S3ClientConf { return fmt::format( "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " - "need_override_endpoint={}, cred_provider_type={}, role_arn={}, external_id={})", - hide_access_key(ak), token.empty() ? "" : "***", endpoint, region, bucket, - max_connections, - request_timeout_ms, connect_timeout_ms, use_virtual_addressing, - need_override_endpoint, cred_provider_type, role_arn, - external_id.empty() ? "" : "***"); + "cred_provider_type={},role_arn={}, external_id={}", + hide_access_key(ak), token, endpoint, region, bucket, max_connections, + request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, + role_arn, external_id); } }; diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index b7e635c1f1d29f..e4e39590892a53 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include "gmock/gmock.h" #include "io/fs/s3_obj_storage_client.h" @@ -35,6 +37,10 @@ class MockS3Client : public Aws::S3::S3Client { MOCK_METHOD(Aws::S3::Model::ListObjectsV2Outcome, ListObjectsV2, (const Aws::S3::Model::ListObjectsV2Request& request), (const, override)); + MOCK_METHOD(Aws::S3::Model::PutObjectOutcome, PutObject, + (const Aws::S3::Model::PutObjectRequest& request), (const, override)); + MOCK_METHOD(Aws::S3::Model::UploadPartOutcome, UploadPart, + (const Aws::S3::Model::UploadPartRequest& request), (const, override)); }; class S3ObjStorageClientMockTest : public testing::Test { @@ -120,9 +126,40 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { files.clear(); } +TEST_F(S3ObjStorageClientMockTest, content_md5_depends_on_s3_express) { + const auto verify_requests = [](bool is_s3_express, bool expect_content_md5) { + auto mock_s3_client = std::make_shared(); + S3ObjStorageClient s3_obj_storage_client(mock_s3_client, is_s3_express); + + EXPECT_CALL(*mock_s3_client, PutObject(testing::_)) + .WillOnce([expect_content_md5](const PutObjectRequest& request) { + EXPECT_EQ(expect_content_md5, request.ContentMD5HasBeenSet()); + return PutObjectOutcome(PutObjectResult {}); + }); + auto put_response = s3_obj_storage_client.put_object( + {.bucket = "dummy-bucket", .key = "content-md5-put"}, "put-body"); + EXPECT_EQ(ErrorCode::OK, put_response.status.code); + + EXPECT_CALL(*mock_s3_client, UploadPart(testing::_)) + .WillOnce([expect_content_md5](const UploadPartRequest& request) { + EXPECT_EQ(expect_content_md5, request.ContentMD5HasBeenSet()); + return UploadPartOutcome(UploadPartResult {}); + }); + auto upload_response = s3_obj_storage_client.upload_part( + {.bucket = "dummy-bucket", + .key = "content-md5-multipart", + .upload_id = "upload-id"}, + "part-body", 1); + EXPECT_EQ(ErrorCode::OK, upload_response.resp.status.code); + }; + + verify_requests(false, true); + verify_requests(true, false); +} + TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { auto path = doris::get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); LOG(INFO) << "config:" << config::ca_cert_file_paths << " path:" << path; ASSERT_FALSE(path.empty()); } -} // namespace doris::io \ No newline at end of file +} // namespace doris::io diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index cb9da739899557..f18111df2e55cf 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -81,44 +81,21 @@ TEST_F(S3UTILTest, is_s3_express_context) { EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com")); EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", - "bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com")); + "bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1." + "amazonaws.com")); + EXPECT_TRUE(is_s3_express("bucket--cnn1-az1--x-s3", + "https://s3.cn-north-1.amazonaws.com.cn")); + + // S3-compatible services must retain their existing endpoint and checksum behavior, + // even if a bucket happens to use the Directory Bucket suffix. EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com")); - EXPECT_FALSE(is_s3_express("bucket--x-s3-suffix", "s3.us-east-1.amazonaws.com")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", + "https://s3.us-east-1.amazonaws.com.example.com")); + EXPECT_FALSE(is_s3_express("bucket--x-s3", "https://s3.us-east-1.amazonaws.com")); + EXPECT_FALSE(is_s3_express("bucket--zone--x-s3", "https://s3.us-east-1.amazonaws.com")); EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path")); } -TEST_F(S3UTILTest, s3_directory_bucket_capabilities) { - auto capabilities = resolve_s3_bucket_capabilities( - "analytics--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com"); - EXPECT_TRUE(capabilities.is_directory_bucket()); - EXPECT_TRUE(capabilities.official_aws_service); - EXPECT_EQ(S3EndpointMode::AWS_SDK_RULES, capabilities.endpoint_mode); - EXPECT_EQ(S3ChecksumPolicy::CRC32C, capabilities.checksum_policy); - EXPECT_FALSE(capabilities.supports_start_after); - EXPECT_FALSE(capabilities.list_is_lexicographic); - EXPECT_FALSE(capabilities.supports_versioning); - EXPECT_FALSE(capabilities.supports_presign); - - capabilities = resolve_s3_bucket_capabilities("analytics--use1-az4--x-s3", ""); - EXPECT_TRUE(capabilities.is_directory_bucket()); - EXPECT_EQ("use1-az4", s3_directory_bucket_zone_id("analytics--use1-az4--x-s3")); - EXPECT_EQ("use1-az4", s3express_endpoint_zone_id( - "analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1." - "amazonaws.com")); - EXPECT_EQ("us-east-1", aws_s3_endpoint_region("s3.us-east-1.amazonaws.com")); - - capabilities = resolve_s3_bucket_capabilities( - "analytics--use1-az4--x-s3", "https://s3.dualstack.us-east-1.amazonaws.com"); - EXPECT_TRUE(capabilities.is_directory_bucket()); - EXPECT_EQ(S3EndpointMode::EXPLICIT_OVERRIDE, capabilities.endpoint_mode); - - capabilities = resolve_s3_bucket_capabilities("analytics--use1-az4--x-s3", - "https://minio.example.com"); - EXPECT_FALSE(capabilities.is_directory_bucket()); - EXPECT_FALSE(capabilities.official_aws_service); - EXPECT_EQ(S3ChecksumPolicy::CONTENT_MD5, capabilities.checksum_policy); -} - // Verifies that check_s3_rate_limiter_config_changed() rebuilds the global GET rate // limiter when the related configs change. This is the behavior the cloud vault refresh // thread relies on to apply dynamically modified s3_get_* rate limiter configs without diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 179ab9679b05d2..2e19ebbb8a3bff 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -38,7 +38,6 @@ #include "common/network_util.h" #include "common/stats.h" #include "common/string_util.h" -#include "cpp/s3_bucket_capabilities.h" #include "cpp/sync_point.h" #include "meta-service/meta_service.h" #include "meta-service/meta_service_helper.h" @@ -769,21 +768,6 @@ static bool use_credential_provider(const ObjectStoreInfoPB& obj) { return obj.has_cred_provider_type() || has_non_empty_role_arn(obj); } -static bool is_unsupported_directory_bucket_vault(const ObjectStoreInfoPB& obj) { - return obj.has_provider() && obj.provider() == ObjectStoreInfoPB::S3 && - resolve_s3_bucket_capabilities(obj.bucket(), obj.endpoint()).is_directory_bucket(); -} - -static int reject_directory_bucket_vault(const ObjectStoreInfoPB& obj, MetaServiceCode& code, - std::string& msg) { - if (!is_unsupported_directory_bucket_vault(obj)) { - return 0; - } - code = MetaServiceCode::INVALID_ARGUMENT; - msg = "S3 Express One Zone is not supported by Cloud Storage Vault in this release"; - return -1; -} - static void create_object_info_with_encrypt(const InstanceInfoPB& instance, ObjectStoreInfoPB* obj, bool sse_enabled, MetaServiceCode& code, std::string& msg) { @@ -797,10 +781,6 @@ static void create_object_info_with_encrypt(const InstanceInfoPB& instance, Obje std::string external_endpoint = obj->has_external_endpoint() ? obj->external_endpoint() : ""; std::string region = obj->has_region() ? obj->region() : ""; - if (reject_directory_bucket_vault(*obj, code, msg) != 0) { - return; - } - if (obj->has_role_arn()) { if (obj->role_arn().empty() || !obj->has_cred_provider_type() || !obj->has_provider() || obj->provider() != ObjectStoreInfoPB::S3 || bucket.empty() || endpoint.empty() || @@ -1125,9 +1105,6 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt msg = ss.str(); return -1; } - if (reject_directory_bucket_vault(new_vault.obj_info(), code, msg) != 0) { - return -1; - } auto origin_vault_info = new_vault.DebugString(); @@ -1271,10 +1248,6 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, const auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info(); - if (reject_directory_bucket_vault(obj, code, msg) != 0) { - return -1; - } - // obj size > 1k, refuse if (obj.ByteSizeLong() > 1024) { code = MetaServiceCode::INVALID_ARGUMENT; diff --git a/common/cpp/aws_logger.h b/common/cpp/aws_logger.h index 9f06cec80fac0f..85734d13e1411c 100644 --- a/common/cpp/aws_logger.h +++ b/common/cpp/aws_logger.h @@ -20,8 +20,6 @@ #include #include #include -#include -#include class DorisAWSLogger final : public Aws::Utils::Logging::LogSystemInterface { public: @@ -38,13 +36,6 @@ class DorisAWSLogger final : public Aws::Utils::Logging::LogSystemInterface { _log_impl(log_level, tag, message_stream.str().c_str()); } - void vaLog(Aws::Utils::Logging::LogLevel log_level, const char* tag, const char* format_str, - va_list args) final { - char buf[4096]; - vsnprintf(buf, sizeof(buf), format_str, args); - _log_impl(log_level, tag, buf); - } - void Flush() final {} private: diff --git a/common/cpp/s3_bucket_capabilities.h b/common/cpp/s3_bucket_capabilities.h deleted file mode 100644 index 999354a195895a..00000000000000 --- a/common/cpp/s3_bucket_capabilities.h +++ /dev/null @@ -1,200 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace doris { - -enum class S3BucketType { GENERAL_PURPOSE, DIRECTORY }; -enum class S3EndpointMode { EXPLICIT_OVERRIDE, AWS_SDK_RULES }; -enum class S3ChecksumPolicy { CONTENT_MD5, CRC32C }; - -struct S3BucketCapabilities { - S3BucketType bucket_type = S3BucketType::GENERAL_PURPOSE; - S3EndpointMode endpoint_mode = S3EndpointMode::EXPLICIT_OVERRIDE; - S3ChecksumPolicy checksum_policy = S3ChecksumPolicy::CONTENT_MD5; - bool official_aws_service = false; - bool require_https = false; - bool require_virtual_addressing = false; - bool supports_start_after = true; - bool list_is_lexicographic = true; - bool supports_versioning = true; - bool supports_presign = true; - - bool is_directory_bucket() const { return bucket_type == S3BucketType::DIRECTORY; } -}; - -inline std::string s3_endpoint_host(std::string_view endpoint) { - const auto scheme = endpoint.find("://"); - if (scheme != std::string_view::npos) { - endpoint.remove_prefix(scheme + 3); - } - endpoint = endpoint.substr(0, endpoint.find_first_of("/?#")); - if (const auto port = endpoint.find(':'); port != std::string_view::npos) { - endpoint = endpoint.substr(0, port); - } - std::string host(endpoint); - std::transform(host.begin(), host.end(), host.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - while (!host.empty() && host.back() == '.') { - host.pop_back(); - } - return host; -} - -inline bool is_aws_s3_endpoint(std::string_view endpoint) { - if (endpoint.empty()) { - // An omitted endpoint means that the AWS SDK resolves the public AWS S3 endpoint - // from the configured region. - return true; - } - const std::string host = s3_endpoint_host(endpoint); - if (host.empty()) { - return false; - } - const bool aws_dns = host.ends_with(".amazonaws.com") || - host.ends_with(".amazonaws.com.cn") || host.ends_with(".api.aws"); - if (!aws_dns) { - return false; - } - return host.starts_with("s3.") || host.starts_with("s3-") || - host.starts_with("s3express-") || host.starts_with("s3express-control.") || - host.find(".s3.") != std::string::npos || - host.find(".s3-") != std::string::npos || - host.find(".s3express-") != std::string::npos; -} - -inline bool is_s3express_control_endpoint(std::string_view endpoint) { - const std::string host = s3_endpoint_host(endpoint); - return host.starts_with("s3express-control.") || - host.find(".s3express-control.") != std::string::npos; -} - -inline bool is_s3express_zonal_endpoint(std::string_view endpoint) { - if (is_s3express_control_endpoint(endpoint)) { - return false; - } - const std::string host = s3_endpoint_host(endpoint); - return host.starts_with("s3express-") || host.find(".s3express-") != std::string::npos; -} - -inline bool is_aws_explicit_s3_endpoint(std::string_view endpoint) { - const std::string host = s3_endpoint_host(endpoint); - return host.find("fips") != std::string::npos || host.find("dualstack") != std::string::npos || - host.find("accelerate") != std::string::npos || - host.find("vpce") != std::string::npos || - is_s3express_control_endpoint(endpoint); -} - -inline std::string s3_directory_bucket_zone_id(std::string_view bucket) { - constexpr std::string_view suffix = "--x-s3"; - if (!bucket.ends_with(suffix)) { - return {}; - } - bucket.remove_suffix(suffix.size()); - const auto separator = bucket.rfind("--"); - return separator == std::string_view::npos ? "" : std::string(bucket.substr(separator + 2)); -} - -inline std::string s3express_endpoint_zone_id(std::string_view endpoint) { - if (is_s3express_control_endpoint(endpoint)) { - return {}; - } - const std::string host = s3_endpoint_host(endpoint); - const auto marker = host.find("s3express-"); - if (marker == std::string::npos) { - return {}; - } - const auto begin = marker + std::string_view("s3express-").size(); - const auto end = host.find('.', begin); - return host.substr(begin, end == std::string::npos ? end : end - begin); -} - -inline std::string s3_endpoint_host_label(std::string_view host, std::size_t begin) { - const auto end = host.find('.', begin); - return std::string(host.substr(begin, end == std::string_view::npos ? end : end - begin)); -} - -inline std::string aws_s3_endpoint_region(std::string_view endpoint) { - const std::string host = s3_endpoint_host(endpoint); - if (host.empty() || host == "s3.amazonaws.com") { - return {}; - } - - const auto control = host.find("s3express-control."); - if (control != std::string::npos) { - const auto begin = control + std::string_view("s3express-control.").size(); - return s3_endpoint_host_label(host, begin); - } - const auto express = host.find("s3express-"); - if (express != std::string::npos) { - const auto dot = host.find('.', express); - return dot == std::string::npos ? "" : s3_endpoint_host_label(host, dot + 1); - } - const auto standard = host.find("s3."); - if (standard != std::string::npos) { - auto begin = standard + std::string_view("s3.").size(); - if (host.substr(begin).starts_with("dualstack.")) { - begin += std::string_view("dualstack.").size(); - } - const auto value = s3_endpoint_host_label(host, begin); - return value == "amazonaws" ? "" : value; - } - if (host.starts_with("s3-")) { - const auto begin = std::string_view("s3-").size(); - return s3_endpoint_host_label(host, begin); - } - return {}; -} - -inline bool is_s3_directory_bucket_name(std::string_view bucket) { - static const std::regex pattern( - "^[a-z0-9]([a-z0-9-]*[a-z0-9])?--[a-z0-9]+-az[0-9]+--x-s3$"); - return bucket.size() >= 3 && bucket.size() <= 63 && - std::regex_match(bucket.begin(), bucket.end(), pattern); -} - -inline S3BucketCapabilities resolve_s3_bucket_capabilities(std::string_view bucket, - std::string_view endpoint) { - S3BucketCapabilities capabilities; - capabilities.official_aws_service = is_aws_s3_endpoint(endpoint); - if (!is_s3_directory_bucket_name(bucket) || !capabilities.official_aws_service) { - return capabilities; - } - - capabilities.bucket_type = S3BucketType::DIRECTORY; - capabilities.endpoint_mode = is_aws_explicit_s3_endpoint(endpoint) - ? S3EndpointMode::EXPLICIT_OVERRIDE - : S3EndpointMode::AWS_SDK_RULES; - capabilities.checksum_policy = S3ChecksumPolicy::CRC32C; - capabilities.require_https = true; - capabilities.require_virtual_addressing = true; - capabilities.supports_start_after = false; - capabilities.list_is_lexicographic = false; - capabilities.supports_versioning = false; - capabilities.supports_presign = false; - return capabilities; -} - -} // namespace doris diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java index bb4713ffdd55d4..ae31eb815d11da 100644 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3FileReader.java @@ -50,10 +50,6 @@ public S3FileReader(String accessKey, String secretKey, String endpoint, String this.region = region; S3Utils.parseURI(uri); this.bucketName = S3Utils.getBucket(); - if (S3Utils.isAwsDirectoryBucket(bucketName, endpoint)) { - throw new IOException( - "S3 Express One Zone is not supported by the Avro Hadoop S3A reader"); - } this.key = S3Utils.getKey(); this.accessKey = accessKey; this.secretKey = secretKey; diff --git a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java index 06a9d0d8654f2b..45845af3c0392e 100644 --- a/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java +++ b/fe/be-java-extensions/avro-scanner/src/main/java/org/apache/doris/avro/S3Utils.java @@ -20,14 +20,8 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Locale; -import java.util.regex.Pattern; public class S3Utils { - private static final Pattern DIRECTORY_BUCKET_PATTERN = Pattern.compile( - "^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9]+-az[0-9]+--x-s3$"); private static final String SCHEMA_S3 = "s3"; private static final String SCHEMA_HTTP = "http"; private static final String SCHEMA_HTTPS = "https"; @@ -105,28 +99,4 @@ public static String getKey() { return key; } - public static boolean isAwsDirectoryBucket(String bucketName, String endpoint) { - if (bucketName == null || !DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches()) { - return false; - } - if (StringUtils.isBlank(endpoint)) { - return true; - } - String value = endpoint.contains(SCHEME_DELIM) ? endpoint : "https://" + endpoint; - try { - String host = new URI(value).getHost(); - if (host == null) { - return false; - } - host = host.toLowerCase(Locale.ROOT); - boolean awsDns = host.endsWith(".amazonaws.com") - || host.endsWith(".amazonaws.com.cn") || host.endsWith(".api.aws"); - return awsDns && (host.startsWith("s3.") || host.startsWith("s3-") - || host.startsWith("s3express-") || host.contains(".s3.") - || host.contains(".s3-") || host.contains(".s3express-")); - } catch (URISyntaxException e) { - return false; - } - } - } diff --git a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java b/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java deleted file mode 100644 index a331c6447bc4ca..00000000000000 --- a/fe/be-java-extensions/avro-scanner/src/test/java/org/apache/doris/avro/S3UtilsTest.java +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.avro; - -import org.junit.Assert; -import org.junit.Test; - -public class S3UtilsTest { - - @Test - public void testDirectoryBucketClassification() { - String bucket = "analytics--use1-az4--x-s3"; - Assert.assertTrue(S3Utils.isAwsDirectoryBucket( - bucket, "https://s3.us-east-1.amazonaws.com")); - Assert.assertTrue(S3Utils.isAwsDirectoryBucket(bucket, "")); - Assert.assertFalse(S3Utils.isAwsDirectoryBucket( - bucket, "https://minio.example.com")); - Assert.assertFalse(S3Utils.isAwsDirectoryBucket( - "analytics--x-s3", "https://s3.us-east-1.amazonaws.com")); - } -} diff --git a/fe/fe-connector/fe-connector-hive/pom.xml b/fe/fe-connector/fe-connector-hive/pom.xml index cb5649dd9ba310..055331c269c4de 100644 --- a/fe/fe-connector/fe-connector-hive/pom.xml +++ b/fe/fe-connector/fe-connector-hive/pom.xml @@ -54,12 +54,6 @@ under the License. ${project.version}     - - ${project.groupId} - fe-filesystem-api - ${project.version} - - ${project.groupId} diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java index f9eb7c0a85c679..2baae9a79926d3 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveScanPlanProvider.java @@ -27,7 +27,6 @@ import org.apache.doris.connector.api.scan.ConnectorScanRangeType; import org.apache.doris.connector.hms.HmsClient; import org.apache.doris.connector.hms.HmsPartitionInfo; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -226,7 +225,6 @@ private void listAndSplitFiles(Configuration conf, PartitionScanInfo partition, HiveFileFormat fileFormat, boolean splittable, long targetSplitSize, List ranges) throws IOException { - rejectUnsupportedDirectoryBucket(partition.location, catalogProperties); Path partPath = new Path(partition.location); FileSystem fs = FileSystem.get(partPath.toUri(), conf); FileStatus[] statuses; @@ -251,17 +249,6 @@ private void listAndSplitFiles(Configuration conf, } } - static void rejectUnsupportedDirectoryBucket(String location, - Map properties) throws IOException { - String endpoint = properties.getOrDefault("AWS_ENDPOINT", - properties.getOrDefault("s3.endpoint", - properties.getOrDefault("fs.s3a.endpoint", properties.get("endpoint")))); - if (S3BucketCapabilities.isDirectoryBucketUri(location, endpoint)) { - throw new IOException( - "S3 Express One Zone is not supported by Hive Hadoop S3A in this release"); - } - } - /** * Splits a file into scan ranges based on target split size. */ diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java deleted file mode 100644 index a5ad0401adf6c3..00000000000000 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveScanPlanProviderTest.java +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.connector.hive; - -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; - -class HiveScanPlanProviderTest { - - @Test - void rejectsAwsDirectoryBucketBeforeHadoopFileSystemAccess() { - assertThrows(IOException.class, () -> HiveScanPlanProvider.rejectUnsupportedDirectoryBucket( - "s3a://analytics--use1-az4--x-s3/table", Collections.emptyMap())); - } - - @Test - void doesNotClassifyCustomS3CompatibleBucketAsDirectoryBucket() { - assertDoesNotThrow(() -> HiveScanPlanProvider.rejectUnsupportedDirectoryBucket( - "s3a://analytics--use1-az4--x-s3/table", - Map.of("fs.s3a.endpoint", "https://minio.example.com"))); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index d733b47525fb6d..31de8d1313e404 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -22,7 +22,6 @@ import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.DatasourcePrintableMap; import org.apache.doris.datasource.property.storage.S3Properties; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.ObjFileSystem; import org.apache.doris.filesystem.spi.ObjStorage; @@ -106,24 +105,12 @@ protected void setProperties(ImmutableMap newProperties) throws // the endpoint for ping need add uri scheme. String pingEndpoint = properties.get(S3Properties.ENDPOINT); if (!pingEndpoint.startsWith("http://") && !pingEndpoint.startsWith("https://")) { - S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( - properties.get(S3Properties.BUCKET), pingEndpoint); - pingEndpoint = (capabilities.isDirectoryBucket() ? "https://" : "http://") - + properties.get(S3Properties.ENDPOINT); + pingEndpoint = "http://" + properties.get(S3Properties.ENDPOINT); properties.put(S3Properties.ENDPOINT, pingEndpoint); properties.put(S3Properties.Env.ENDPOINT, pingEndpoint); } String region = S3Properties.getRegionOfEndpoint(pingEndpoint); properties.putIfAbsent(S3Properties.REGION, region); - try { - S3BucketCapabilities.resolve(properties.get(S3Properties.BUCKET), pingEndpoint) - .validateDirectoryConfiguration(pingEndpoint, - properties.get(S3Properties.REGION), - Boolean.parseBoolean(properties.getOrDefault( - S3Properties.USE_PATH_STYLE, "false"))); - } catch (IllegalArgumentException e) { - throw new DdlException(e.getMessage()); - } if (needCheck) { String bucketName = properties.get(S3Properties.BUCKET); @@ -179,20 +166,9 @@ protected static void pingS3(String bucketName, String rootPath, Map expectedKey.equals(object.getKey())); - continuationToken = remoteObjects.isTruncated() - ? remoteObjects.getContinuationToken() : null; - } while (!found && continuationToken != null); - if (!found) { - throw new IOException("ListObjects did not return the ping object"); - } + org.apache.doris.filesystem.spi.RemoteObjects remoteObjects = + objStorage.listObjects(testObj, null); + LOG.info("remoteObjects: {}", remoteObjects); } catch (IOException e) { throw new DdlException("pingS3 failed(list)," + " please check your endpoint, ak/sk or permissions" @@ -211,8 +187,8 @@ protected static void pingS3(String bucketName, String rootPath, Map(newProperties, "=", true, false, true, false), e); + + new DatasourcePrintableMap<>(newProperties, "=", true, false, true, false)); } try { @@ -352,3 +328,4 @@ protected void getProcNodeData(BaseProcResult result) { readUnlock(); } } + diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index abba037a641822..1b3158b561a7d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -27,7 +27,6 @@ import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand; import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; import org.apache.doris.rpc.BackendServiceProxy; @@ -150,12 +149,6 @@ private void updateDefaultStorageVaultCache(Pair newDefaultVault private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map properties, String name) throws Exception { Cloud.ObjectStoreInfoPB.Builder objBuilder = S3Properties.getObjStoreInfoPB(properties); - if (objBuilder.hasProvider() && objBuilder.getProvider() == Cloud.ObjectStoreInfoPB.Provider.S3 - && S3BucketCapabilities.resolve(objBuilder.getBucket(), objBuilder.getEndpoint()) - .isDirectoryBucket()) { - throw new DdlException( - "S3 Express One Zone is not supported by Cloud Storage Vault in this release"); - } Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder(); alterObjVaultBuilder.setName(name); alterObjVaultBuilder.setObjInfo(objBuilder.build()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java index 1f24f9207fc9d7..63ab2081110409 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java @@ -18,7 +18,6 @@ package org.apache.doris.common.util; import org.apache.doris.common.UserException; -import org.apache.doris.filesystem.S3BucketCapabilities; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; @@ -77,6 +76,9 @@ public class S3URI { private static final Set OS_SCHEMES = ImmutableSet.of("s3", "s3a", "s3n", "bos", "oss", "cos", "cosn", "gs", "obs", "azure"); + /** Suffix of S3Express storage bucket names. */ + private static final String S3_DIRECTORY_BUCKET_SUFFIX = "--x-s3"; + private URI uri; private String bucket; @@ -354,7 +356,28 @@ public boolean useS3DirectoryBucket() { * @return true if the bucket name indicates the bucket is a directory bucket */ public static boolean isS3DirectoryBucket(final String bucketName) { - return S3BucketCapabilities.isDirectoryBucketName(bucketName); + if (bucketName == null || !bucketName.endsWith(S3_DIRECTORY_BUCKET_SUFFIX)) { + return false; + } + // Check if the bucket name has the correct format: bucket-name--azid--x-s3 + // The bucket name should have at least 3 segments separated by "--" + String[] segments = bucketName.split("--"); + if (segments.length < 3) { + return false; + } + // The last segment should be "x-s3" + if (!"x-s3".equals(segments[segments.length - 1])) { + return false; + } + // The second-to-last segment should be the availability zone identifier + // It should have a format like "usw2-az1", "use1-az4", etc. + String azid = segments[segments.length - 2]; + if (azid == null || azid.isEmpty()) { + return false; + } + // Basic validation: azid should contain at least one hyphen and not be empty + // More sophisticated validation could be added here if needed + return azid.contains("-") && azid.length() > 3; } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index c8eb72958bb4a7..15ca49a51297b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -22,7 +22,6 @@ import org.apache.doris.common.InternalErrorCode; import org.apache.doris.common.UserException; import org.apache.doris.common.credentials.CloudCredential; -import org.apache.doris.filesystem.S3BucketCapabilities; import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; @@ -48,7 +47,6 @@ import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; @@ -60,6 +58,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -93,8 +92,37 @@ private static AwsCredentialsProvider getAwsCredencialsProvider(CloudCredential @Deprecated public static S3Client buildS3Client(URI endpoint, String region, CloudCredential credential, boolean isUsePathStyle) { - return buildS3Client(endpoint, region, isUsePathStyle, - getAwsCredencialsProvider(credential), null); + EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy + .builder() + .baseDelay(Duration.ofSeconds(1)) + .maxBackoffTime(Duration.ofMinutes(1)) + .build(); + // retry 3 time with Equal backoff + RetryPolicy retryPolicy = RetryPolicy + .builder() + .numRetries(3) + .backoffStrategy(backoffStrategy) + .build(); + ClientOverrideConfiguration clientConf = ClientOverrideConfiguration + .builder() + // set retry policy + .retryPolicy(retryPolicy) + // using AwsS3V4Signer + .putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()) + .build(); + return S3Client.builder() + .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) + .connectionTimeout(Duration.ofSeconds(30)).build()) + .endpointOverride(endpoint) + .credentialsProvider(getAwsCredencialsProvider(credential)) + .region(Region.of(region)) + .overrideConfiguration(clientConf) + // disable chunkedEncoding because of bos not supported + .serviceConfiguration(S3Configuration.builder() + .chunkedEncodingEnabled(false) + .pathStyleAccessEnabled(isUsePathStyle) + .build()) + .build(); } /** @@ -190,15 +218,6 @@ private static AwsCredentialsProvider getAwsCredencialsProvider(URI endpoint, St public static S3Client buildS3Client(URI endpoint, String region, boolean isUsePathStyle, AwsCredentialsProvider credential) { - return buildS3Client(endpoint, region, isUsePathStyle, credential, null); - } - - public static S3Client buildS3Client(URI endpoint, String region, boolean isUsePathStyle, - AwsCredentialsProvider credential, String bucket) { - S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( - bucket, endpoint == null ? null : endpoint.toString()); - capabilities.validateDirectoryConfiguration( - endpoint == null ? null : endpoint.toString(), region, isUsePathStyle); EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy .builder() .baseDelay(Duration.ofSeconds(1)) @@ -210,36 +229,62 @@ public static S3Client buildS3Client(URI endpoint, String region, boolean isUseP .numRetries(3) .backoffStrategy(backoffStrategy) .build(); - ClientOverrideConfiguration.Builder clientConf = ClientOverrideConfiguration - .builder().retryPolicy(retryPolicy); - if (!capabilities.isDirectoryBucket()) { - clientConf.putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()); - } - S3ClientBuilder builder = S3Client.builder() + ClientOverrideConfiguration clientConf = ClientOverrideConfiguration + .builder() + // set retry policy + .retryPolicy(retryPolicy) + // using AwsS3V4Signer + .putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()) + .build(); + return S3Client.builder() .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) .connectionTimeout(Duration.ofSeconds(30)).build()) + .endpointOverride(endpoint) .credentialsProvider(credential) .region(Region.of(region)) - .disableS3ExpressSessionAuth(!capabilities.isDirectoryBucket() - && !capabilities.officialAwsService()) - .overrideConfiguration(clientConf.build()) + .overrideConfiguration(clientConf) // disable chunkedEncoding because of bos not supported .serviceConfiguration(S3Configuration.builder() .chunkedEncodingEnabled(false) - .pathStyleAccessEnabled(capabilities.isDirectoryBucket() - ? false : isUsePathStyle) - .build()); - if (!capabilities.isDirectoryBucket() && endpoint != null) { - builder.endpointOverride(endpoint); - } - return builder.build(); + .pathStyleAccessEnabled(isUsePathStyle) + .build()) + .build(); } public static S3Client buildS3Client(URI endpoint, String region, boolean isUsePathStyle, String accessKey, String secretKey, String sessionToken, String roleArn, String externalId) { - return buildS3Client(endpoint, region, isUsePathStyle, - getAwsCredencialsProvider(endpoint, region, accessKey, secretKey, - sessionToken, roleArn, externalId), null); + EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy + .builder() + .baseDelay(Duration.ofSeconds(1)) + .maxBackoffTime(Duration.ofMinutes(1)) + .build(); + // retry 3 time with Equal backoff + RetryPolicy retryPolicy = RetryPolicy + .builder() + .numRetries(3) + .backoffStrategy(backoffStrategy) + .build(); + ClientOverrideConfiguration clientConf = ClientOverrideConfiguration + .builder() + // set retry policy + .retryPolicy(retryPolicy) + // using AwsS3V4Signer + .putAdvancedOption(SdkAdvancedClientOption.SIGNER, AwsS3V4Signer.create()) + .build(); + return S3Client.builder() + .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) + .connectionTimeout(Duration.ofSeconds(30)).build()) + .endpointOverride(endpoint) + .credentialsProvider(getAwsCredencialsProvider(endpoint, region, accessKey, secretKey, + sessionToken, roleArn, externalId)) + .region(Region.of(region)) + .overrideConfiguration(clientConf) + // disable chunkedEncoding because of bos not supported + .serviceConfiguration(S3Configuration.builder() + .chunkedEncodingEnabled(false) + .pathStyleAccessEnabled(isUsePathStyle) + .build()) + .build(); } public static String getLongestPrefix(String globPattern) { @@ -258,6 +303,47 @@ public static String getLongestPrefix(String globPattern) { return globPattern.substring(0, earliestSpecialCharIndex); } + public static boolean isExactS3ExpressObject(String path, String endpoint) throws UserException { + if (!(path.regionMatches(true, 0, "s3://", 0, "s3://".length()) + || path.regionMatches(true, 0, "s3a://", 0, "s3a://".length()) + || path.regionMatches(true, 0, "s3n://", 0, "s3n://".length())) + || !getLongestPrefix(path).equals(path)) { + return false; + } + return isS3Express(S3URI.create(path).getBucket(), endpoint); + } + + private static boolean isS3Express(String bucketName, String endpoint) { + if (!isDirectoryBucketName(bucketName) || Strings.isNullOrEmpty(endpoint)) { + return false; + } + String endpointUri = endpoint.contains("://") ? endpoint : "https://" + endpoint; + String host = URI.create(endpointUri).getHost(); + if (host == null) { + return false; + } + host = host.toLowerCase(Locale.ROOT); + boolean awsDns = host.endsWith(".amazonaws.com") + || host.endsWith(".amazonaws.com.cn") + || host.endsWith(".api.aws"); + return awsDns && (host.startsWith("s3.") + || host.startsWith("s3-") + || host.startsWith("s3express-") + || host.contains(".s3.") + || host.contains(".s3-") + || host.contains(".s3express-")); + } + + private static boolean isDirectoryBucketName(String bucketName) { + String suffix = "--x-s3"; + if (bucketName == null || !bucketName.endsWith(suffix)) { + return false; + } + String prefix = bucketName.substring(0, bucketName.length() - suffix.length()); + int zoneSeparator = prefix.lastIndexOf("--"); + return zoneSeparator > 0 && prefix.substring(zoneSeparator + 2).contains("-az"); + } + // Apply some rules to extend the globs parsing behavior public static String extendGlobs(String pathPattern) { return extendGlobNumberRange(pathPattern); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java index 5ec5e91a33ac03..551448dc767a72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java @@ -65,7 +65,7 @@ public void testFeConnection() throws Exception { URI.create(endpoint), properties.getRegion(), Boolean.parseBoolean(properties.getUsePathStyle()), - properties.getAwsCredentialsProvider(), bucket)) { + properties.getAwsCredentialsProvider())) { client.headBucket(b -> b.bucket(bucket)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java index 8ac706ad1336fb..0fa6a21f59efc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java @@ -1869,18 +1869,8 @@ private void objCommit(Executor fileSystemExecutor, List> a + "/" + s3MPUPendingUpload.getKey(); try { objFs.completeMultipartUpload(remotePath, s3MPUPendingUpload.getUploadId(), - s3MPUPendingUpload.getEtags(), - s3MPUPendingUpload.isSetChecksumAlgorithm() - ? s3MPUPendingUpload.getChecksumAlgorithm().name() : null, - s3MPUPendingUpload.isSetPartChecksums() - ? s3MPUPendingUpload.getPartChecksums() : java.util.Map.of()); + s3MPUPendingUpload.getEtags()); } catch (java.io.IOException e) { - try { - objFs.getObjStorage().abortMultipartUpload( - remotePath, s3MPUPendingUpload.getUploadId()); - } catch (java.io.IOException abortFailure) { - e.addSuppressed(abortFailure); - } throw new RuntimeException("Failed to complete MPU for " + remotePath, e); } uncompletedMpuPendingUploads.remove(new UncompletedMpuPendingUpload(s3MPUPendingUpload, path)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 9b4fdbab2598d9..e7c6f9a64a0920 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -40,7 +40,6 @@ import org.apache.doris.datasource.hudi.HudiSchemaCacheValue; import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.datasource.mvcc.MvccUtil; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; @@ -174,18 +173,9 @@ protected void doInitialize() throws UserException { long tableMetaStartTime = System.currentTimeMillis(); try { - String tableLocation = hmsTable.getRemoteTable().getSd().getLocation(); - Map storageProperties = hmsTable.getStoragePropertiesMap(); - String endpoint = storageProperties.getOrDefault("AWS_ENDPOINT", - storageProperties.getOrDefault("s3.endpoint", - storageProperties.get("fs.s3a.endpoint"))); - if (S3BucketCapabilities.isDirectoryBucketUri(tableLocation, endpoint)) { - throw new UserException( - "S3 Express One Zone is not supported by Hudi Hadoop S3A in this release"); - } hudiClient = hmsTable.getHudiClient(); hudiClient.reloadActiveTimeline(); - basePath = tableLocation; + basePath = hmsTable.getRemoteTable().getSd().getLocation(); inputFormat = hmsTable.getRemoteTable().getSd().getInputFormat(); serdeLib = hmsTable.getRemoteTable().getSd().getSerdeInfo().getSerializationLib(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java index 0aeae737c875d0..2662dc55ff7fd5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/source/LakeSoulScanNode.java @@ -25,7 +25,6 @@ import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.lakesoul.LakeSoulExternalTable; import org.apache.doris.datasource.lakesoul.LakeSoulUtils; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanContext; import org.apache.doris.qe.SessionVariable; @@ -95,14 +94,6 @@ protected void doInitialize() throws UserException { lakeSoulExternalTable = (LakeSoulExternalTable) desc.getTable(); TableInfo tableInfo = lakeSoulExternalTable.getLakeSoulTableInfo(); location = tableInfo.getTablePath(); - Map catalogProperties = lakeSoulExternalTable.getCatalog().getProperties(); - String endpoint = catalogProperties.getOrDefault("AWS_ENDPOINT", - catalogProperties.getOrDefault("s3.endpoint", - catalogProperties.get("fs.s3a.endpoint"))); - if (S3BucketCapabilities.isDirectoryBucketUri(location, endpoint)) { - throw new UserException( - "S3 Express One Zone is not supported by LakeSoul object storage in this release"); - } tableName = tableInfo.getTableName(); partitions = tableInfo.getPartitions(); readType = LakeSoulOptions.ReadType$.MODULE$.FULL_READ(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index dcbfff104adf3c..9c9631a516fdac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -18,9 +18,7 @@ package org.apache.doris.datasource.property.metastore; import org.apache.doris.common.security.authentication.ExecutionAuthenticator; -import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.foundation.property.ConnectorProperty; import com.google.common.collect.ImmutableList; @@ -62,19 +60,6 @@ protected AbstractPaimonProperties(Map props) { public abstract Catalog initializeCatalog(String catalogName, List storagePropertiesList); - protected void rejectDirectoryBucketStorage(List storagePropertiesList) { - String endpoint = storagePropertiesList.stream() - .filter(S3Properties.class::isInstance) - .map(S3Properties.class::cast) - .map(S3Properties::getEndpoint) - .findFirst() - .orElse(""); - if (S3BucketCapabilities.isDirectoryBucketUri(warehouse, endpoint)) { - throw new IllegalArgumentException( - "S3 Express One Zone is not supported by Paimon storage in this release"); - } - } - protected void appendCatalogOptions() { if (StringUtils.isNotBlank(warehouse)) { catalogOptions.set(CatalogOptions.WAREHOUSE.key(), warehouse); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java index 74d008b499379e..9bc77d543d3d59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStoreProperties.java @@ -85,7 +85,6 @@ private HiveConf buildHiveConf() { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { - rejectDirectoryBucketStorage(storagePropertiesList); HiveConf hiveConf = buildHiveConf(); buildCatalogOptions(); StorageProperties ossProps = storagePropertiesList.stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java index ccb8d778a83a52..df0ebae97490a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java @@ -38,7 +38,6 @@ protected PaimonFileSystemMetaStoreProperties(Map props) { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { - rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); Configuration conf = new Configuration(); storagePropertiesList.forEach(storageProperties -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java index 048ac15a48db5c..e7e6689d3e3cab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java @@ -87,7 +87,6 @@ private Configuration buildHiveConfiguration(List storageProp @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { - rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); Configuration conf = buildHiveConfiguration(storagePropertiesList); appendUserHadoopConfig(conf); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java index 3472f977051196..7568d59c5fed33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonJdbcMetaStoreProperties.java @@ -110,7 +110,6 @@ protected void checkRequiredProperties() { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { - rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); Configuration conf = new Configuration(); for (StorageProperties storageProperties : storagePropertiesList) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java index 3a1c410b6371e0..465fc873b7c5d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonRestMetaStoreProperties.java @@ -77,7 +77,6 @@ public String getPaimonCatalogType() { @Override public Catalog initializeCatalog(String catalogName, List storagePropertiesList) { - rejectDirectoryBucketStorage(storagePropertiesList); buildCatalogOptions(); CatalogContext catalogContext = CatalogContext.create(catalogOptions); return CatalogFactory.createCatalog(catalogContext); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java index c2b9be5db9fccd..ebfd7f7119b3d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java @@ -24,6 +24,9 @@ import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; +import org.apache.doris.common.util.S3Util; +import org.apache.doris.datasource.property.storage.ObjectStorageProperties; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileSystem; import org.apache.doris.filesystem.GlobListing; @@ -96,6 +99,7 @@ protected void getAllFileStatus() throws UserException { fileStatusList.add(fileStatuses); } } else { + StorageProperties storageProperties = brokerDesc.getStorageProperties(); for (BrokerFileGroup fileGroup : fileGroups) { long groupFileSize = 0; List fileStatuses = Lists.newArrayList(); @@ -106,12 +110,20 @@ protected void getAllFileStatus() throws UserException { // Plain listFiles uses S3 prefix matching which can return unintended // prefix-siblings (e.g. "file.csv.bz2" when listing "file.csv"). List entries; - try { - GlobListing listing = fs.globListWithLimit( - Location.of(path), null, 0, 0); - entries = listing.getFiles(); - } catch (UnsupportedOperationException ex) { - entries = fs.listFiles(Location.of(path)); + if (storageProperties instanceof ObjectStorageProperties + && S3Util.isExactS3ExpressObject(path, + ((ObjectStorageProperties) storageProperties).getEndpoint())) { + Location location = Location.of(path); + entries = List.of(new FileEntry( + location, fs.newInputFile(location).length(), false, 0L, List.of())); + } else { + try { + GlobListing listing = fs.globListWithLimit( + Location.of(path), null, 0, 0); + entries = listing.getFiles(); + } catch (UnsupportedOperationException ex) { + entries = fs.listFiles(Location.of(path)); + } } for (FileEntry e : entries) { fileStatuses.add(new TBrokerFileStatus( diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 6c7962e7267033..695a4b8fc5c903 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -166,14 +166,22 @@ protected void parseFile() throws AnalysisException { } try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(brokerDesc)) { List entries; - // Always prefer glob semantics: for exact paths it ensures precise matching - // (prevents S3 prefix-based listing from including unintended files like - // "file.csv.bz2" when listing "file.csv"). Fall back to listFiles only - // when the filesystem does not support glob. - try { - entries = fs.globListWithLimit(Location.of(path), "", 0, 0).getFiles(); - } catch (UnsupportedOperationException ex) { - entries = fs.listFiles(Location.of(path)); + if (sp instanceof ObjectStorageProperties + && S3Util.isExactS3ExpressObject( + path, ((ObjectStorageProperties) sp).getEndpoint())) { + Location location = Location.of(path); + entries = List.of(new FileEntry( + location, fs.newInputFile(location).length(), false, 0L, List.of())); + } else { + // Always prefer glob semantics: for exact paths it ensures precise matching + // (prevents S3 prefix-based listing from including unintended files like + // "file.csv.bz2" when listing "file.csv"). Fall back to listFiles only + // when the filesystem does not support glob. + try { + entries = fs.globListWithLimit(Location.of(path), "", 0, 0).getFiles(); + } catch (UnsupportedOperationException ex) { + entries = fs.listFiles(Location.of(path)); + } } for (FileEntry e : entries) { fileStatuses.add(new TBrokerFileStatus( diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java index 4b976ed86cd3c5..4f0dd06df7e80f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java @@ -25,6 +25,19 @@ public class S3UtilTest { + @Test + public void testIsExactS3ExpressObject() throws Exception { + String object = "s3://analytics--use1-az4--x-s3/data/file.parquet"; + String awsEndpoint = "https://s3.us-east-1.amazonaws.com"; + + Assert.assertTrue(S3Util.isExactS3ExpressObject(object, awsEndpoint)); + Assert.assertFalse(S3Util.isExactS3ExpressObject( + "s3://analytics--use1-az4--x-s3/data/*.parquet", awsEndpoint)); + Assert.assertFalse(S3Util.isExactS3ExpressObject(object, "https://minio.example.com")); + Assert.assertFalse(S3Util.isExactS3ExpressObject( + "s3://analytics/data/file.parquet", awsEndpoint)); + } + @Test public void testExtendGlobNumberRange_simpleRange() { // Test simple range expansion {1..3} @@ -457,4 +470,3 @@ public void testExpandBracketPatterns_malformedBracket() { Assert.assertEquals("file[abc.csv", S3Util.expandBracketPatterns("file[abc.csv")); } } - diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java deleted file mode 100644 index 8150841283740c..00000000000000 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java +++ /dev/null @@ -1,277 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.filesystem; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Locale; -import java.util.regex.Pattern; - -/** Request-level S3 capabilities derived from the bucket and service endpoint. */ -public final class S3BucketCapabilities { - - public enum BucketType { - GENERAL_PURPOSE, - DIRECTORY - } - - public enum EndpointMode { - EXPLICIT_OVERRIDE, - AWS_SDK_RULES - } - - public enum ChecksumPolicy { - CONTENT_MD5, - CRC32C - } - - private static final Pattern DIRECTORY_BUCKET_PATTERN = Pattern.compile( - "^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9]+-az[0-9]+--x-s3$"); - - private final BucketType bucketType; - private final EndpointMode endpointMode; - private final ChecksumPolicy checksumPolicy; - private final boolean officialAwsService; - private final String directoryBucketZone; - - private S3BucketCapabilities(BucketType bucketType, EndpointMode endpointMode, - ChecksumPolicy checksumPolicy, boolean officialAwsService, - String directoryBucketZone) { - this.bucketType = bucketType; - this.endpointMode = endpointMode; - this.checksumPolicy = checksumPolicy; - this.officialAwsService = officialAwsService; - this.directoryBucketZone = directoryBucketZone; - } - - public static S3BucketCapabilities resolve(String bucket, String endpoint) { - boolean officialAws = isAwsS3Endpoint(endpoint); - if (!isDirectoryBucketName(bucket) || !officialAws) { - return new S3BucketCapabilities(BucketType.GENERAL_PURPOSE, - EndpointMode.EXPLICIT_OVERRIDE, ChecksumPolicy.CONTENT_MD5, officialAws, ""); - } - EndpointMode mode = isExplicitAwsEndpoint(endpoint) - ? EndpointMode.EXPLICIT_OVERRIDE : EndpointMode.AWS_SDK_RULES; - return new S3BucketCapabilities(BucketType.DIRECTORY, mode, - ChecksumPolicy.CRC32C, true, bucketZone(bucket)); - } - - public static boolean isDirectoryBucketName(String bucket) { - return bucket != null && bucket.length() >= 3 && bucket.length() <= 63 - && DIRECTORY_BUCKET_PATTERN.matcher(bucket).matches(); - } - - /** Returns true when an S3-style URI targets an AWS Directory Bucket. */ - public static boolean isDirectoryBucketUri(String location, String endpoint) { - if (location == null || location.isBlank()) { - return false; - } - try { - URI uri = new URI(location); - String scheme = uri.getScheme(); - if (scheme == null || !(scheme.equalsIgnoreCase("s3") - || scheme.equalsIgnoreCase("s3a") || scheme.equalsIgnoreCase("s3n"))) { - return false; - } - String bucket = uri.getHost() == null ? uri.getAuthority() : uri.getHost(); - return resolve(bucket, endpoint).isDirectoryBucket(); - } catch (URISyntaxException e) { - return false; - } - } - - public static boolean isAwsS3Endpoint(String endpoint) { - if (endpoint == null || endpoint.isBlank()) { - // No override means that the AWS SDK resolves the public AWS endpoint from region. - return true; - } - String host = endpointHost(endpoint); - if (host.isEmpty()) { - return false; - } - boolean awsDns = host.endsWith(".amazonaws.com") - || host.endsWith(".amazonaws.com.cn") || host.endsWith(".api.aws"); - if (!awsDns) { - return false; - } - return host.startsWith("s3.") || host.startsWith("s3-") - || host.startsWith("s3express-") || host.startsWith("s3express-control.") - || host.contains(".s3.") || host.contains(".s3-") - || host.contains(".s3express-"); - } - - public static boolean isS3ExpressControlEndpoint(String endpoint) { - String host = endpointHost(endpoint); - return host.startsWith("s3express-control.") || host.contains(".s3express-control."); - } - - public static boolean isS3ExpressZonalEndpoint(String endpoint) { - if (isS3ExpressControlEndpoint(endpoint)) { - return false; - } - String host = endpointHost(endpoint); - return host.startsWith("s3express-") || host.contains(".s3express-"); - } - - public void validateDirectoryConfiguration(String endpoint, String region, boolean usePathStyle) { - if (officialAwsService && isS3ExpressControlEndpoint(endpoint)) { - throw new IllegalArgumentException( - "s3express-control endpoint cannot be used for object operations"); - } - if (officialAwsService && isS3ExpressZonalEndpoint(endpoint) && !isDirectoryBucket()) { - throw new IllegalArgumentException( - "An S3 Express endpoint requires a valid AWS Directory Bucket name"); - } - if (!isDirectoryBucket()) { - return; - } - if (region == null || region.isBlank()) { - throw new IllegalArgumentException("AWS Directory Bucket requires AWS_REGION"); - } - if (usePathStyle) { - throw new IllegalArgumentException( - "Path-style addressing is not supported for AWS Directory Bucket"); - } - if (endpoint != null && endpoint.toLowerCase(Locale.ROOT).startsWith("http://")) { - throw new IllegalArgumentException("AWS Directory Bucket requires HTTPS"); - } - if (endpointMode != EndpointMode.AWS_SDK_RULES) { - throw new IllegalArgumentException( - "AWS Directory Bucket requires a standard regional or zonal S3 endpoint"); - } - String endpointRegion = endpointRegion(endpoint); - if (!endpointRegion.isEmpty() && !endpointRegion.equals(region)) { - throw new IllegalArgumentException("Configured endpoint region " + endpointRegion - + " does not match AWS_REGION " + region + " for Directory Bucket"); - } - String endpointZone = endpointZone(endpoint); - if (!endpointZone.isEmpty() && !endpointZone.equals(directoryBucketZone)) { - throw new IllegalArgumentException("Configured endpoint zone " + endpointZone - + " does not match Directory Bucket zone " + directoryBucketZone); - } - } - - public BucketType bucketType() { - return bucketType; - } - - public EndpointMode endpointMode() { - return endpointMode; - } - - public ChecksumPolicy checksumPolicy() { - return checksumPolicy; - } - - public boolean officialAwsService() { - return officialAwsService; - } - - public boolean isDirectoryBucket() { - return bucketType == BucketType.DIRECTORY; - } - - public boolean supportsStartAfter() { - return !isDirectoryBucket(); - } - - public boolean listIsLexicographic() { - return !isDirectoryBucket(); - } - - public boolean supportsVersioning() { - return !isDirectoryBucket(); - } - - public boolean supportsPresign() { - return !isDirectoryBucket(); - } - - private static boolean isExplicitAwsEndpoint(String endpoint) { - String host = endpointHost(endpoint); - return host.contains("fips") || host.contains("dualstack") - || host.contains("accelerate") || host.contains("vpce") - || isS3ExpressControlEndpoint(endpoint); - } - - private static String bucketZone(String bucket) { - if (bucket == null || !bucket.endsWith("--x-s3")) { - return ""; - } - int separator = bucket.lastIndexOf("--", bucket.length() - "--x-s3".length() - 1); - return separator < 0 ? "" : bucket.substring(separator + 2, bucket.length() - "--x-s3".length()); - } - - private static String endpointZone(String endpoint) { - String host = endpointHost(endpoint); - int marker = host.indexOf("s3express-"); - if (marker < 0 || host.startsWith("s3express-control", marker)) { - return ""; - } - int begin = marker + "s3express-".length(); - int end = host.indexOf('.', begin); - return end < 0 ? host.substring(begin) : host.substring(begin, end); - } - - private static String endpointRegion(String endpoint) { - String host = endpointHost(endpoint); - if (host.isEmpty() || "s3.amazonaws.com".equals(host)) { - return ""; - } - int control = host.indexOf("s3express-control."); - if (control >= 0) { - return nextHostLabel(host, control + "s3express-control.".length()); - } - int express = host.indexOf("s3express-"); - if (express >= 0) { - int dot = host.indexOf('.', express); - return dot < 0 ? "" : nextHostLabel(host, dot + 1); - } - int standard = host.indexOf("s3."); - if (standard >= 0) { - int begin = standard + "s3.".length(); - if (host.startsWith("dualstack.", begin)) { - begin += "dualstack.".length(); - } - String value = nextHostLabel(host, begin); - return "amazonaws".equals(value) ? "" : value; - } - if (host.startsWith("s3-")) { - return nextHostLabel(host, "s3-".length()); - } - return ""; - } - - private static String nextHostLabel(String host, int begin) { - int end = host.indexOf('.', begin); - return end < 0 ? host.substring(begin) : host.substring(begin, end); - } - - private static String endpointHost(String endpoint) { - if (endpoint == null || endpoint.isBlank()) { - return ""; - } - String value = endpoint.contains("://") ? endpoint : "https://" + endpoint; - try { - String host = new URI(value).getHost(); - return host == null ? "" : host.toLowerCase(Locale.ROOT); - } catch (URISyntaxException e) { - return ""; - } - } -} diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java index 77293bc5265013..9b2d88dc0776f6 100644 --- a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java @@ -27,16 +27,10 @@ public final class UploadPartResult { private final int partNumber; private final String etag; - private final String checksumCrc32c; public UploadPartResult(int partNumber, String etag) { - this(partNumber, etag, null); - } - - public UploadPartResult(int partNumber, String etag, String checksumCrc32c) { this.partNumber = partNumber; this.etag = etag; - this.checksumCrc32c = checksumCrc32c; } public int partNumber() { @@ -46,8 +40,4 @@ public int partNumber() { public String etag() { return etag; } - - public String checksumCrc32c() { - return checksumCrc32c; - } } diff --git a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java deleted file mode 100644 index 4f51be7104f808..00000000000000 --- a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/S3BucketCapabilitiesTest.java +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.filesystem; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -class S3BucketCapabilitiesTest { - - @Test - void resolvesOfficialDirectoryBucket() { - S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( - "analytics--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com"); - - Assertions.assertTrue(capabilities.isDirectoryBucket()); - Assertions.assertTrue(capabilities.officialAwsService()); - Assertions.assertEquals(S3BucketCapabilities.EndpointMode.AWS_SDK_RULES, - capabilities.endpointMode()); - Assertions.assertEquals(S3BucketCapabilities.ChecksumPolicy.CRC32C, - capabilities.checksumPolicy()); - Assertions.assertFalse(capabilities.supportsStartAfter()); - Assertions.assertFalse(capabilities.listIsLexicographic()); - Assertions.assertFalse(capabilities.supportsVersioning()); - Assertions.assertFalse(capabilities.supportsPresign()); - } - - @Test - void keepsCustomEndpointGeneralPurpose() { - S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( - "analytics--use1-az4--x-s3", "https://minio.example.com"); - - Assertions.assertFalse(capabilities.isDirectoryBucket()); - Assertions.assertFalse(capabilities.officialAwsService()); - Assertions.assertEquals(S3BucketCapabilities.ChecksumPolicy.CONTENT_MD5, - capabilities.checksumPolicy()); - } - - @Test - void rejectsUnsupportedDirectoryConfiguration() { - S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( - "analytics--use1-az4--x-s3", "https://s3.dualstack.us-east-1.amazonaws.com"); - - Assertions.assertThrows(IllegalArgumentException.class, - () -> capabilities.validateDirectoryConfiguration( - "https://s3.dualstack.us-east-1.amazonaws.com", "us-east-1", false)); - } - - @Test - void validatesRegionAndZone() { - S3BucketCapabilities capabilities = S3BucketCapabilities.resolve( - "analytics--use1-az4--x-s3", - "https://analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com"); - - Assertions.assertDoesNotThrow(() -> capabilities.validateDirectoryConfiguration( - "https://analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com", - "us-east-1", false)); - Assertions.assertThrows(IllegalArgumentException.class, - () -> capabilities.validateDirectoryConfiguration( - "https://analytics--use1-az4--x-s3.s3express-use1-az4.us-east-1.amazonaws.com", - "us-west-2", false)); - - S3BucketCapabilities wrongZone = S3BucketCapabilities.resolve( - "analytics--use1-az4--x-s3", - "https://analytics--use1-az4--x-s3.s3express-use1-az5.us-east-1.amazonaws.com"); - Assertions.assertThrows(IllegalArgumentException.class, - () -> wrongZone.validateDirectoryConfiguration( - "https://analytics--use1-az4--x-s3.s3express-use1-az5.us-east-1.amazonaws.com", - "us-east-1", false)); - } - - @Test - void recognizesSdkRoutedDirectoryBucketUri() { - Assertions.assertTrue(S3BucketCapabilities.isDirectoryBucketUri( - "s3a://analytics--use1-az4--x-s3/warehouse", "")); - Assertions.assertFalse(S3BucketCapabilities.isDirectoryBucketUri( - "s3a://analytics--use1-az4--x-s3/warehouse", "https://minio.example.com")); - } -} diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java index 52620fa8140922..9ae077801c4f91 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/main/java/org/apache/doris/filesystem/hdfs/DFSFileSystem.java @@ -23,7 +23,6 @@ import org.apache.doris.filesystem.FileIterator; import org.apache.doris.filesystem.GlobListing; import org.apache.doris.filesystem.Location; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.spi.HadoopAuthenticator; import org.apache.hadoop.conf.Configuration; @@ -80,13 +79,6 @@ private org.apache.hadoop.fs.FileSystem getHadoopFs(Path path) throws IOExceptio if (closed.get()) { throw new IOException("DFSFileSystem is closed."); } - String endpoint = properties.getOrDefault("AWS_ENDPOINT", - properties.getOrDefault("s3.endpoint", - properties.getOrDefault("fs.s3a.endpoint", properties.get("endpoint")))); - if (S3BucketCapabilities.isDirectoryBucketUri(path.toString(), endpoint)) { - throw new IOException( - "S3 Express One Zone is not supported by Hadoop S3A in this release"); - } String authority = path.toUri().getAuthority(); String key = authority != null ? authority : ""; org.apache.hadoop.fs.FileSystem fs = fsByAuthority.get(key); diff --git a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java index e7d390004ae71e..54abb2974acf54 100644 --- a/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-hdfs/src/test/java/org/apache/doris/filesystem/hdfs/DFSFileSystemTest.java @@ -69,29 +69,6 @@ void constructor_succeedsWithHdfsProperties() { Assertions.assertDoesNotThrow(() -> new DFSFileSystem(props)); } - @Test - void rejectsDirectoryBucketBeforeHadoopS3aAccess() { - DFSFileSystem fs = new DFSFileSystem(new HashMap<>()); - - IOException exception = Assertions.assertThrows(IOException.class, - () -> fs.requireFs(new Path( - "s3a://analytics--use1-az4--x-s3/warehouse/table"))); - Assertions.assertTrue(exception.getMessage().contains("Hadoop S3A")); - } - - @Test - void keepsCustomS3CompatibleBucketOnHadoopPath() throws Exception { - Map properties = Map.of( - "fs.s3a.endpoint", "https://minio.example.com"); - DFSFileSystem fs = new DFSFileSystem(properties); - org.apache.hadoop.fs.FileSystem hadoopFs = - Mockito.mock(org.apache.hadoop.fs.FileSystem.class); - injectHadoopFs(fs, "analytics--use1-az4--x-s3", hadoopFs); - - Assertions.assertSame(hadoopFs, fs.requireFs(new Path( - "s3a://analytics--use1-az4--x-s3/warehouse/table"))); - } - // ------------------------------------------------------------------ // close() and post-close behavior // ------------------------------------------------------------------ diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java index d9ddf2368f23ac..0f6eeb35608d97 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java @@ -17,20 +17,10 @@ package org.apache.doris.filesystem.s3; -import org.apache.doris.filesystem.FileEntry; -import org.apache.doris.filesystem.GlobListing; -import org.apache.doris.filesystem.Location; -import org.apache.doris.filesystem.spi.ObjectStorageUri; -import org.apache.doris.filesystem.spi.RemoteObject; -import org.apache.doris.filesystem.spi.RemoteObjects; import org.apache.doris.filesystem.spi.S3CompatibleFileSystem; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.Optional; -import java.util.regex.Pattern; /** * S3 filesystem backed by the AWS S3 SDK. @@ -38,7 +28,6 @@ public class S3FileSystem extends S3CompatibleFileSystem { private final S3FileSystemProperties properties; - private final S3ObjStorage s3Storage; public S3FileSystem(S3FileSystemProperties properties) { this(properties, new S3ObjStorage(properties)); @@ -47,83 +36,44 @@ public S3FileSystem(S3FileSystemProperties properties) { S3FileSystem(S3FileSystemProperties properties, S3ObjStorage objStorage) { super(objStorage, objStorage.isUsePathStyle()); this.properties = properties; - this.s3Storage = objStorage; } public S3FileSystem(S3ObjStorage objStorage) { super(objStorage, objStorage.isUsePathStyle()); this.properties = null; - this.s3Storage = objStorage; } public Optional properties() { return Optional.ofNullable(properties); } - private static String slashTerminatedNonGlobPrefix(String globPattern) { - String prefix = longestNonGlobPrefix(globPattern); - if (prefix.isEmpty() || prefix.endsWith("/")) { - return prefix; + @Override + protected String globListPrefix(String globPattern) { + if (isDirectoryBucketEndpoint()) { + return slashTerminatedNonGlobPrefix(globPattern); } - int slash = prefix.lastIndexOf('/'); - return slash < 0 ? "" : prefix.substring(0, slash + 1); + return super.globListPrefix(globPattern); } @Override - protected boolean restartListingAfterDelete(String bucket) { - return s3Storage.isDirectoryBucket(bucket); + protected List globListPrefixes(String globPattern, String listPrefix) { + if (isDirectoryBucketEndpoint()) { + return List.of(listPrefix); + } + return super.globListPrefixes(globPattern, listPrefix); } - @Override - public GlobListing globListWithLimit(Location path, String startAfter, long maxBytes, - long maxFiles) throws IOException { - String uri = path.uri(); - ObjectStorageUri parsed = parseUri(uri); - if (!s3Storage.isDirectoryBucket(parsed.bucket())) { - return super.globListWithLimit(path, startAfter, maxBytes, maxFiles); - } + private boolean isDirectoryBucketEndpoint() { + return properties != null && properties.isDirectoryBucketEndpoint(); + } - String keyPattern = expandNumericRanges(parsed.key()); - Pattern matcher = Pattern.compile(globToRegex(keyPattern)); - String listPrefix = slashTerminatedNonGlobPrefix(keyPattern); - String base = uriBase(uri, parsed); - String listUri = base + listPrefix; - - List matches = new ArrayList<>(); - String continuationToken = null; - do { - RemoteObjects response = s3Storage.listObjects(listUri, continuationToken); - for (RemoteObject object : response.getObjectList()) { - if (!object.getKey().endsWith("/") - && matcher.matcher(object.getKey()).matches() - && (startAfter == null - || compareUtf8Binary(object.getKey(), startAfter) > 0)) { - matches.add(object); - } - } - continuationToken = response.isTruncated() - ? response.getContinuationToken() : null; - } while (continuationToken != null); - - matches.sort(Comparator.comparing(RemoteObject::getKey, S3FileSystem::compareUtf8Binary)); - List files = new ArrayList<>(); - long totalSize = 0L; - int nextIndex = matches.size(); - for (int i = 0; i < matches.size(); i++) { - RemoteObject object = matches.get(i); - files.add(new FileEntry(Location.of(base + object.getKey()), object.getSize(), false, - object.getModificationTime(), List.of())); - totalSize += object.getSize(); - if ((maxFiles > 0 && files.size() >= maxFiles) - || (maxBytes > 0 && totalSize >= maxBytes)) { - nextIndex = i + 1; - break; - } + private static String slashTerminatedNonGlobPrefix(String globPattern) { + String prefix = longestNonGlobPrefix(globPattern); + if (prefix.isEmpty() || prefix.endsWith("/")) { + return prefix; } - String maxFile = matches.isEmpty() ? "" - : nextIndex < matches.size() ? matches.get(nextIndex).getKey() - : matches.get(Math.min(files.size(), matches.size()) - 1).getKey(); - return new GlobListing(files, parsed.bucket(), listPrefix, maxFile); + int slash = prefix.lastIndexOf('/'); + return slash < 0 ? "" : prefix.substring(0, slash + 1); } protected static boolean isSingleLevelGlob(String pathStr) { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java index 03a85051d59041..34e8fb54b934bd 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java @@ -18,7 +18,6 @@ package org.apache.doris.filesystem.s3; import org.apache.doris.filesystem.FileSystemType; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.properties.BackendStorageKind; import org.apache.doris.filesystem.properties.BackendStorageProperties; import org.apache.doris.filesystem.properties.FileSystemProperties; @@ -212,8 +211,6 @@ public void validate() { .check(this::hasInvalidUsePathStyle, "use_path_style must be true or false, got: '" + getUsePathStyle() + "'") .validate("Invalid S3 filesystem properties"); - capabilitiesFor(bucket).validateDirectoryConfiguration( - endpoint, region, Boolean.parseBoolean(usePathStyle)); } @Override @@ -286,10 +283,6 @@ public Map toMap() { @Override public Map toHadoopConfigurationMap() { - if (capabilitiesFor(bucket).isDirectoryBucket()) { - throw new IllegalArgumentException( - "S3 Express One Zone is not supported by Hadoop S3A in this release"); - } Map cfg = new HashMap<>(); cfg.put("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); cfg.put("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); @@ -334,8 +327,9 @@ public boolean hasAssumeRole() { return StringUtils.isNotBlank(roleArn); } - public S3BucketCapabilities capabilitiesFor(String requestBucket) { - return S3BucketCapabilities.resolve(requestBucket, endpoint); + public boolean isDirectoryBucketEndpoint() { + return StringUtils.containsIgnoreCase(endpoint, "s3express-control.") + || StringUtils.containsIgnoreCase(endpoint, "s3express-"); } private static void putIfNotBlank(Map map, String key, String value) { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index 7d4f484395eeb6..ce2616432a0ebb 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -17,7 +17,6 @@ package org.apache.doris.filesystem.s3; -import org.apache.doris.filesystem.S3BucketCapabilities; import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.ObjStorage; import org.apache.doris.filesystem.spi.ObjectListOptions; @@ -38,7 +37,6 @@ import software.amazon.awssdk.services.s3.S3ClientBuilder; import software.amazon.awssdk.services.s3.S3Configuration; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; import software.amazon.awssdk.services.s3.model.CompletedPart; @@ -78,6 +76,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -100,8 +99,8 @@ public class S3ObjStorage implements ObjStorage { /** Bucket name; may be null if not provided (listObjectsWithPrefix and related methods will fail). */ private final String bucket; private final AtomicBoolean closed = new AtomicBoolean(false); - private volatile S3Client generalClient; - private volatile S3Client directoryClient; + private volatile S3Client client; + private volatile S3Client expressClient; private volatile AwsCredentialsProvider credentialsProvider; public S3ObjStorage(S3FileSystemProperties properties) { @@ -128,14 +127,14 @@ public S3Client getClient() throws IOException { if (closed.get()) { throw new IOException("S3ObjStorage is already closed"); } - if (generalClient == null) { + if (client == null) { synchronized (this) { - if (generalClient == null) { - generalClient = buildClient(); + if (client == null) { + client = buildClient(); } } } - return generalClient; + return client; } protected S3Client buildClient() throws IOException { @@ -145,13 +144,15 @@ protected S3Client buildClient() throws IOException { getCredentialsProvider(), false); } - protected S3Client buildDirectoryClient() throws IOException { - return buildClient(s3Properties.getEndpoint(), s3Properties.getRegion(), + protected S3Client buildExpressClient() throws IOException { + return buildClient( + s3Properties.getEndpoint(), + s3Properties.getRegion(), getCredentialsProvider(), true); } private S3Client buildClient(String endpointStr, String region, - AwsCredentialsProvider clientCredentialsProvider, boolean directory) + AwsCredentialsProvider clientCredentialsProvider, boolean express) throws IOException { S3ClientBuilder builder = S3Client.builder() .httpClient(UrlConnectionHttpClient.builder() @@ -160,16 +161,13 @@ private S3Client buildClient(String endpointStr, String region, .build()) .credentialsProvider(clientCredentialsProvider) .region(Region.of(region)) - .disableS3ExpressSessionAuth(!directory - && !S3BucketCapabilities.isAwsS3Endpoint(endpointStr)) + .disableS3ExpressSessionAuth(!express) .serviceConfiguration(S3Configuration.builder() .chunkedEncodingEnabled(false) - .pathStyleAccessEnabled(directory ? false : usePathStyle) + .pathStyleAccessEnabled(express ? false : usePathStyle) .build()); - // Preserve the configured override for the general-purpose client. Directory Bucket - // requests use the separate SDK-routed client and never enter this branch. - if (!directory && StringUtils.isNotBlank(endpointStr)) { + if (!express && StringUtils.isNotBlank(endpointStr)) { if (!endpointStr.contains("://")) { endpointStr = "https://" + endpointStr; } @@ -205,36 +203,21 @@ private AwsCredentialsProvider getCredentialsProvider() { return credentialsProvider; } - S3BucketCapabilities capabilitiesFor(String requestBucket) { - return S3BucketCapabilities.resolve(requestBucket, s3Properties.getEndpoint()); - } - - boolean isDirectoryBucket(String requestBucket) { - return capabilitiesFor(requestBucket).isDirectoryBucket(); - } - private S3Client clientFor(String requestBucket) throws IOException { - S3BucketCapabilities capabilities = capabilitiesFor(requestBucket); - if (!capabilities.isDirectoryBucket()) { + if (!isS3Express(requestBucket, s3Properties.getEndpoint())) { return getClient(); } - try { - capabilities.validateDirectoryConfiguration(s3Properties.getEndpoint(), - s3Properties.getRegion(), usePathStyle); - } catch (IllegalArgumentException e) { - throw new IOException(e.getMessage(), e); - } if (closed.get()) { throw new IOException("S3ObjStorage is already closed"); } - if (directoryClient == null) { + if (expressClient == null) { synchronized (this) { - if (directoryClient == null) { - directoryClient = buildDirectoryClient(); + if (expressClient == null) { + expressClient = buildExpressClient(); } } } - return directoryClient; + return expressClient; } private AwsCredentialsProvider buildStsSourceCredentialsProvider() { @@ -258,42 +241,32 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); - S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); - String requestPrefix = capabilities.isDirectoryBucket() - ? directoryListPrefix(uri.key()) : uri.key(); ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() .bucket(uri.bucket()) - .prefix(requestPrefix); + .prefix(uri.key()); if (options != null) { if (StringUtils.isNotBlank(options.continuationToken())) { builder.continuationToken(options.continuationToken()); - } else if (capabilities.supportsStartAfter() - && StringUtils.isNotBlank(options.startAfter())) { + } else if (StringUtils.isNotBlank(options.startAfter())) { builder.startAfter(options.startAfter()); } if (options.maxKeys() > 0) { builder.maxKeys(options.maxKeys()); } if (StringUtils.isNotBlank(options.delimiter())) { - if (capabilities.isDirectoryBucket() && !"/".equals(options.delimiter())) { - throw new IOException("AWS Directory Bucket only supports '/' as delimiter"); - } builder.delimiter(options.delimiter()); } } try { - List objects = new ArrayList<>(); - ListObjectsV2Response response = clientFor(uri.bucket()).listObjectsV2(builder.build()); - response.contents().stream() - .filter(s3Obj -> s3Obj.key().startsWith(uri.key())) + ListObjectsV2Response response = getClient().listObjectsV2(builder.build()); + List objects = response.contents().stream() .map(s3Obj -> new org.apache.doris.filesystem.spi.RemoteObject( s3Obj.key(), getRelativePath(uri.key(), s3Obj.key()), s3Obj.eTag(), s3Obj.size(), - s3Obj.lastModified() != null - ? s3Obj.lastModified().toEpochMilli() : 0L)) - .forEach(objects::add); + s3Obj.lastModified() != null ? s3Obj.lastModified().toEpochMilli() : 0L)) + .collect(Collectors.toList()); return new RemoteObjects(objects, response.isTruncated(), response.nextContinuationToken()); } catch (SdkException e) { @@ -371,15 +344,9 @@ public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath public void putObject(String remotePath, org.apache.doris.filesystem.spi.RequestBody requestBody) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); - S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); - PutObjectRequest.Builder request = PutObjectRequest.builder() - .bucket(uri.bucket()).key(uri.key()); - if (capabilities.checksumPolicy() == S3BucketCapabilities.ChecksumPolicy.CRC32C) { - request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); - } try (InputStream content = requestBody.content()) { clientFor(uri.bucket()).putObject( - request.build(), + PutObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build(), software.amazon.awssdk.core.sync.RequestBody.fromInputStream( content, requestBody.contentLength())); } catch (SdkException e) { @@ -391,7 +358,7 @@ public void putObject(String remotePath, org.apache.doris.filesystem.spi.Request public void deleteObject(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - clientFor(uri.bucket()).deleteObject(DeleteObjectRequest.builder() + getClient().deleteObject(DeleteObjectRequest.builder() .bucket(uri.bucket()).key(uri.key()).build()); } catch (S3Exception e) { if (e.statusCode() == 404) { @@ -407,11 +374,8 @@ public void deleteObject(String remotePath) throws IOException { public void copyObject(String srcPath, String dstPath) throws IOException { S3Uri srcUri = S3Uri.parse(srcPath, usePathStyle); S3Uri dstUri = S3Uri.parse(dstPath, usePathStyle); - if (isDirectoryBucket(srcUri.bucket()) || isDirectoryBucket(dstUri.bucket())) { - throw new IOException("CopyObject is not supported for AWS Directory Bucket"); - } try { - clientFor(dstUri.bucket()).copyObject(CopyObjectRequest.builder() + getClient().copyObject(CopyObjectRequest.builder() .copySource(SdkHttpUtils.urlEncodeIgnoreSlashes( srcUri.bucket() + "/" + srcUri.key())) .destinationBucket(dstUri.bucket()) @@ -426,15 +390,9 @@ public void copyObject(String srcPath, String dstPath) throws IOException { @Override public String initiateMultipartUpload(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); - S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); - CreateMultipartUploadRequest.Builder request = CreateMultipartUploadRequest.builder() - .bucket(uri.bucket()).key(uri.key()); - if (capabilities.checksumPolicy() == S3BucketCapabilities.ChecksumPolicy.CRC32C) { - request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); - } try { - CreateMultipartUploadResponse response = clientFor(uri.bucket()) - .createMultipartUpload(request.build()); + CreateMultipartUploadResponse response = clientFor(uri.bucket()).createMultipartUpload( + CreateMultipartUploadRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return response.uploadId(); } catch (SdkException e) { throw new IOException("initiateMultipartUpload failed for " + remotePath @@ -446,24 +404,16 @@ public String initiateMultipartUpload(String remotePath) throws IOException { public UploadPartResult uploadPart(String remotePath, String uploadId, int partNum, org.apache.doris.filesystem.spi.RequestBody body) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); - S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); - UploadPartRequest.Builder request = UploadPartRequest.builder() - .bucket(uri.bucket()).key(uri.key()) - .uploadId(uploadId).partNumber(partNum) - .contentLength(body.contentLength()); - if (capabilities.checksumPolicy() == S3BucketCapabilities.ChecksumPolicy.CRC32C) { - request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); - } try (InputStream content = body.content()) { UploadPartResponse response = clientFor(uri.bucket()).uploadPart( - request.build(), + UploadPartRequest.builder() + .bucket(uri.bucket()).key(uri.key()) + .uploadId(uploadId).partNumber(partNum) + .contentLength(body.contentLength()) + .build(), software.amazon.awssdk.core.sync.RequestBody.fromInputStream( content, body.contentLength())); - if (capabilities.isDirectoryBucket() && StringUtils.isBlank(response.checksumCRC32C())) { - throw new IOException("UploadPart response is missing CRC32C for AWS Directory Bucket, part=" - + partNum); - } - return new UploadPartResult(partNum, response.eTag(), response.checksumCRC32C()); + return new UploadPartResult(partNum, response.eTag()); } catch (SdkException e) { throw new IOException("uploadPart " + partNum + " failed for " + remotePath + ": " + e.getMessage(), e); @@ -474,27 +424,9 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN public void completeMultipartUpload(String remotePath, String uploadId, List parts) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); - S3BucketCapabilities capabilities = capabilitiesFor(uri.bucket()); - List sortedParts = parts.stream() - .sorted(java.util.Comparator.comparingInt(UploadPartResult::partNumber)) + List completedParts = parts.stream() + .map(p -> CompletedPart.builder().partNumber(p.partNumber()).eTag(p.etag()).build()) .collect(Collectors.toList()); - List completedParts = new ArrayList<>(sortedParts.size()); - for (int i = 0; i < sortedParts.size(); i++) { - UploadPartResult part = sortedParts.get(i); - if (capabilities.isDirectoryBucket() && part.partNumber() != i + 1) { - throw new IOException("Multipart upload parts must be consecutive from 1"); - } - CompletedPart.Builder completed = CompletedPart.builder() - .partNumber(part.partNumber()).eTag(part.etag()); - if (capabilities.isDirectoryBucket()) { - if (StringUtils.isBlank(part.checksumCrc32c())) { - throw new IOException( - "AWS Directory Bucket multipart completion requires CRC32C for every part"); - } - completed.checksumCRC32C(part.checksumCrc32c()); - } - completedParts.add(completed.build()); - } try { clientFor(uri.bucket()).completeMultipartUpload(CompleteMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId) @@ -513,9 +445,6 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx clientFor(uri.bucket()).abortMultipartUpload(AbortMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId).build()); } catch (S3Exception e) { - if (e.statusCode() == 404) { - return; - } // Re-throw so callers know the abort failed; orphaned parts may still exist // and require manual cleanup or a lifecycle rule. throw new IOException("abortMultipartUpload failed for " + remotePath @@ -613,19 +542,15 @@ public RemoteObjects listObjectsWithPrefix(String prefix, String subPrefix, String continuationToken) throws IOException { requireBucket("listObjectsWithPrefix"); String fullPrefix = normalizeAndCombinePrefix(prefix, subPrefix); - S3BucketCapabilities capabilities = capabilitiesFor(bucket); - String requestPrefix = capabilities.isDirectoryBucket() - ? directoryListPrefix(fullPrefix) : fullPrefix; try { ListObjectsV2Request.Builder reqBuilder = ListObjectsV2Request.builder() .bucket(bucket) - .prefix(requestPrefix); + .prefix(fullPrefix); if (StringUtils.isNotBlank(continuationToken)) { reqBuilder.continuationToken(continuationToken); } - ListObjectsV2Response resp = clientFor(bucket).listObjectsV2(reqBuilder.build()); + ListObjectsV2Response resp = getClient().listObjectsV2(reqBuilder.build()); List files = resp.contents().stream() - .filter(s3Obj -> s3Obj.key().startsWith(fullPrefix)) .map(s3Obj -> new RemoteObject( s3Obj.key(), getRelativePathSafe(prefix, s3Obj.key()), @@ -646,7 +571,7 @@ public RemoteObjects headObjectWithMeta(String prefix, String subKey) throws IOE requireBucket("headObjectWithMeta"); String fullKey = normalizeAndCombinePrefix(prefix, subKey); try { - HeadObjectResponse resp = clientFor(bucket).headObject( + HeadObjectResponse resp = getClient().headObject( HeadObjectRequest.builder() .bucket(bucket) .key(fullKey) @@ -667,9 +592,6 @@ fullKey, getRelativePathSafe(prefix, fullKey), resp.eTag(), resp.contentLength() @Override public String getPresignedUrl(String objectKey) throws IOException { requireBucket("getPresignedUrl"); - if (!capabilitiesFor(bucket).supportsPresign()) { - throw new IOException("Presigned URL is not supported for AWS Directory Bucket"); - } try { PutObjectRequest putReq = PutObjectRequest.builder() .bucket(bucket) @@ -705,14 +627,11 @@ public void deleteObjectsByKeys(String bucket, List keys) throws IOExcep List identifiers = batch.stream() .map(k -> ObjectIdentifier.builder().key(k).build()) .collect(Collectors.toList()); - DeleteObjectsRequest.Builder request = DeleteObjectsRequest.builder() + DeleteObjectsRequest request = DeleteObjectsRequest.builder() .bucket(bucket) - .delete(Delete.builder().objects(identifiers).quiet(true).build()); - if (capabilitiesFor(bucket).checksumPolicy() - == S3BucketCapabilities.ChecksumPolicy.CRC32C) { - request.checksumAlgorithm(ChecksumAlgorithm.CRC32C); - } - DeleteObjectsResponse response = clientFor(bucket).deleteObjects(request.build()); + .delete(Delete.builder().objects(identifiers).quiet(true).build()) + .build(); + DeleteObjectsResponse response = getClient().deleteObjects(request); if (response.hasErrors()) { for (S3Error error : response.errors()) { LOG.warn("Failed to delete object key={} from bucket={}: {} {}", @@ -759,14 +678,6 @@ private static String normalizeAndCombinePrefix(String prefix, String subPrefix) return normalized.isEmpty() ? subPrefix : normalized + subPrefix; } - private static String directoryListPrefix(String prefix) { - if (prefix == null || prefix.isEmpty() || prefix.endsWith("/")) { - return prefix == null ? "" : prefix; - } - int slash = prefix.lastIndexOf('/'); - return slash < 0 ? "" : prefix.substring(0, slash + 1); - } - private static String getRelativePathSafe(String prefix, String key) { String normalized = (prefix == null || prefix.isEmpty()) ? "" : (prefix.endsWith("/") ? prefix : prefix + "/"); @@ -776,17 +687,48 @@ private static String getRelativePathSafe(String prefix, String key) { return key.substring(normalized.length()); } + private static boolean isS3Express(String bucketName, String endpoint) { + if (!isDirectoryBucketName(bucketName) || StringUtils.isBlank(endpoint)) { + return false; + } + String endpointUri = endpoint.contains("://") ? endpoint : "https://" + endpoint; + String host = URI.create(endpointUri).getHost(); + if (host == null) { + return false; + } + host = host.toLowerCase(Locale.ROOT); + boolean awsDns = host.endsWith(".amazonaws.com") + || host.endsWith(".amazonaws.com.cn") + || host.endsWith(".api.aws"); + return awsDns && (host.startsWith("s3.") + || host.startsWith("s3-") + || host.startsWith("s3express-") + || host.contains(".s3.") + || host.contains(".s3-") + || host.contains(".s3express-")); + } + + private static boolean isDirectoryBucketName(String bucketName) { + String suffix = "--x-s3"; + if (bucketName == null || !bucketName.endsWith(suffix)) { + return false; + } + String prefix = bucketName.substring(0, bucketName.length() - suffix.length()); + int zoneSeparator = prefix.lastIndexOf("--"); + return zoneSeparator > 0 && prefix.substring(zoneSeparator + 2).contains("-az"); + } + @Override public void close() throws IOException { if (closed.compareAndSet(false, true)) { - if (generalClient != null) { - generalClient.close(); + if (client != null) { + client.close(); + client = null; } - if (directoryClient != null && directoryClient != generalClient) { - directoryClient.close(); + if (expressClient != null) { + expressClient.close(); + expressClient = null; } - generalClient = null; - directoryClient = null; } } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java index e1cf014d1b151f..e96bcf53b8e4ed 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemPropertiesTest.java @@ -271,18 +271,4 @@ void of_rejectsUnsupportedCredentialsProviderType() { Assertions.assertTrue(exception.getMessage().contains("Invalid S3 filesystem properties")); Assertions.assertTrue(exception.getMessage().contains("Unsupported s3.credentials_provider_type")); } - - @Test - void directoryBucketRejectsHadoopS3AConversion() { - Map raw = new HashMap<>(); - raw.put("s3.endpoint", "https://s3.us-east-1.amazonaws.com"); - raw.put("s3.region", "us-east-1"); - raw.put("s3.bucket", "analytics--use1-az4--x-s3"); - - S3FileSystemProperties properties = S3FileSystemProperties.of(raw); - - IllegalArgumentException exception = Assertions.assertThrows( - IllegalArgumentException.class, properties::toHadoopConfigurationMap); - Assertions.assertTrue(exception.getMessage().contains("Hadoop S3A")); - } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java index c135fbc002b0d0..fe9b28dd8c90b4 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java @@ -678,33 +678,28 @@ void globListWithLimit_paginatesExpandedPrefixesInUtf8BinaryOrder() throws IOExc @Test void globListWithLimit_directoryBucketFallsBackToSlashTerminatedStaticPrefix() throws IOException { S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( - "s3.endpoint", "https://s3.us-east-1.amazonaws.com", - "s3.region", "us-east-1")); + "s3.endpoint", "https://s3express-usw2-az1.us-west-2.amazonaws.com", + "s3.region", "us-west-2")); S3FileSystem directoryBucketFs = new S3FileSystem(properties, mockStorage); - String bucket = "analytics--use1-az4--x-s3"; - Mockito.when(mockStorage.isDirectoryBucket(bucket)).thenReturn(true); Mockito.when(mockStorage.listObjects( - ArgumentMatchers.eq("s3://" + bucket + "/data/"), ArgumentMatchers.isNull())) + ArgumentMatchers.eq("s3://bucket/data/"), ArgumentMatchers.isNull())) .thenReturn(new RemoteObjects( List.of( - new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L), - new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L)), + new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L), + new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L)), false, null)); GlobListing listing = directoryBucketFs.globListWithLimit( - Location.of("s3://" + bucket + "/data/[ab]*.csv"), null, 0L, 1L); + Location.of("s3://bucket/data/[ab]*.csv"), null, 0L, 0L); - Assertions.assertEquals(1, listing.getFiles().size()); - Assertions.assertEquals("s3://" + bucket + "/data/a.csv", - listing.getFiles().get(0).location().uri()); - Assertions.assertEquals("data/b.csv", listing.getMaxFile()); + Assertions.assertEquals(2, listing.getFiles().size()); Assertions.assertEquals("data/", listing.getPrefix()); Mockito.verify(mockStorage).listObjects( - ArgumentMatchers.eq("s3://" + bucket + "/data/"), ArgumentMatchers.isNull()); + ArgumentMatchers.eq("s3://bucket/data/"), ArgumentMatchers.isNull()); Mockito.verify(mockStorage, Mockito.never()).listObjects( - ArgumentMatchers.eq("s3://" + bucket + "/data/a"), ArgumentMatchers.any()); + ArgumentMatchers.eq("s3://bucket/data/a"), ArgumentMatchers.any()); Mockito.verify(mockStorage, Mockito.never()).listObjects( - ArgumentMatchers.eq("s3://" + bucket + "/data/b"), ArgumentMatchers.any()); + ArgumentMatchers.eq("s3://bucket/data/b"), ArgumentMatchers.any()); } @Test diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java index 7f1dd761f9ab96..deefd3925a5855 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java @@ -33,7 +33,6 @@ import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; -import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; @@ -92,87 +91,6 @@ void getClient_returnsInjectedMockClient() throws IOException { Assertions.assertEquals(mockS3, storage.getClient()); } - @Test - void directoryBucketUsesCrc32cForPutAndMultipart() throws IOException { - Map props = new HashMap<>(); - props.put("AWS_ENDPOINT", "https://s3.us-east-1.amazonaws.com"); - props.put("AWS_REGION", "us-east-1"); - props.put("AWS_ACCESS_KEY", "testAK"); - props.put("AWS_SECRET_KEY", "testSK"); - props.put("AWS_BUCKET", "analytics--use1-az4--x-s3"); - storage = new TestableS3ObjStorage(props, mockS3); - - Mockito.when(mockS3.putObject(ArgumentMatchers.any(PutObjectRequest.class), - ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class))) - .thenReturn(PutObjectResponse.builder().build()); - Mockito.when(mockS3.createMultipartUpload( - ArgumentMatchers.any(CreateMultipartUploadRequest.class))) - .thenReturn(CreateMultipartUploadResponse.builder().uploadId("upload-1").build()); - Mockito.when(mockS3.uploadPart(ArgumentMatchers.any(UploadPartRequest.class), - ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class))) - .thenReturn(UploadPartResponse.builder().eTag("etag-1") - .checksumCRC32C("crc32c-1").build()); - Mockito.when(mockS3.completeMultipartUpload( - ArgumentMatchers.any(CompleteMultipartUploadRequest.class))) - .thenReturn(software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse - .builder().build()); - Mockito.when(mockS3.deleteObjects(ArgumentMatchers.any( - software.amazon.awssdk.services.s3.model.DeleteObjectsRequest.class))) - .thenReturn(software.amazon.awssdk.services.s3.model.DeleteObjectsResponse - .builder().build()); - - String path = "s3://analytics--use1-az4--x-s3/data/file"; - storage.putObject(path, RequestBody.of(new ByteArrayInputStream(new byte[] {1}), 1)); - String uploadId = storage.initiateMultipartUpload(path); - UploadPartResult part = storage.uploadPart(path, uploadId, 1, - RequestBody.of(new ByteArrayInputStream(new byte[] {1}), 1)); - storage.completeMultipartUpload(path, uploadId, List.of(part)); - storage.deleteObjectsByKeys("analytics--use1-az4--x-s3", List.of("data/file")); - - ArgumentCaptor putCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); - Mockito.verify(mockS3).putObject(putCaptor.capture(), - ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class)); - Assertions.assertEquals(ChecksumAlgorithm.CRC32C, putCaptor.getValue().checksumAlgorithm()); - ArgumentCaptor createCaptor = - ArgumentCaptor.forClass(CreateMultipartUploadRequest.class); - Mockito.verify(mockS3).createMultipartUpload(createCaptor.capture()); - Assertions.assertEquals(ChecksumAlgorithm.CRC32C, - createCaptor.getValue().checksumAlgorithm()); - Assertions.assertEquals("crc32c-1", part.checksumCrc32c()); - ArgumentCaptor completeCaptor = - ArgumentCaptor.forClass(CompleteMultipartUploadRequest.class); - Mockito.verify(mockS3).completeMultipartUpload(completeCaptor.capture()); - Assertions.assertEquals("crc32c-1", completeCaptor.getValue().multipartUpload() - .parts().get(0).checksumCRC32C()); - ArgumentCaptor deleteCaptor = - ArgumentCaptor.forClass( - software.amazon.awssdk.services.s3.model.DeleteObjectsRequest.class); - Mockito.verify(mockS3).deleteObjects(deleteCaptor.capture()); - Assertions.assertEquals(ChecksumAlgorithm.CRC32C, - deleteCaptor.getValue().checksumAlgorithm()); - } - - @Test - void directoryListOmitsStartAfterAndUsesParentPrefix() throws IOException { - Map props = new HashMap<>(); - props.put("AWS_ENDPOINT", "https://s3.us-east-1.amazonaws.com"); - props.put("AWS_REGION", "us-east-1"); - props.put("AWS_ACCESS_KEY", "testAK"); - props.put("AWS_SECRET_KEY", "testSK"); - storage = new TestableS3ObjStorage(props, mockS3); - Mockito.when(mockS3.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) - .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); - - storage.listObjectsWithOptions("s3://analytics--use1-az4--x-s3/data/file*", - org.apache.doris.filesystem.spi.ObjectListOptions.builder() - .startAfter("data/file0").build()); - - ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); - Mockito.verify(mockS3).listObjectsV2(captor.capture()); - Assertions.assertEquals("data/", captor.getValue().prefix()); - Assertions.assertNull(captor.getValue().startAfter()); - } - // ------------------------------------------------------------------ // listObjects() // ------------------------------------------------------------------ @@ -581,11 +499,6 @@ private static class TestableS3ObjStorage extends S3ObjStorage { protected S3Client buildClient() { return mockClient; } - - @Override - protected S3Client buildDirectoryClient() { - return mockClient; - } } private static class InspectableS3ObjStorage extends TestableS3ObjStorage { diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java index f5c7bae56ca525..f98a9e9640d426 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java @@ -109,18 +109,8 @@ public void deleteObjectsByKeys(String bucket, List keys) throws IOExcep */ public void completeMultipartUpload(String remotePath, String uploadId, Map etags) throws IOException { - completeMultipartUpload(remotePath, uploadId, etags, null, Map.of()); - } - - public void completeMultipartUpload(String remotePath, String uploadId, - Map etags, String checksumAlgorithm, - Map partChecksums) throws IOException { - if (checksumAlgorithm != null && !"CRC32C".equals(checksumAlgorithm)) { - throw new IOException("Unsupported multipart checksum algorithm: " + checksumAlgorithm); - } List parts = etags.entrySet().stream() - .map(e -> new UploadPartResult(e.getKey(), e.getValue(), - partChecksums.get(e.getKey()))) + .map(e -> new UploadPartResult(e.getKey(), e.getValue())) .sorted(Comparator.comparingInt(UploadPartResult::partNumber)) .collect(Collectors.toList()); objStorage.completeMultipartUpload(remotePath, uploadId, parts); diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java index d124ef50d424b9..5746fed5dd8549 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java @@ -195,28 +195,19 @@ private void deleteRecursive(String prefix) throws IOException { String continuationToken = null; do { RemoteObjects batch = objStorage.listObjects(prefix, continuationToken); - boolean deletedObjects = false; if (!batch.getObjectList().isEmpty()) { List keys = new ArrayList<>(batch.getObjectList().size()); for (RemoteObject obj : batch.getObjectList()) { keys.add(obj.getKey()); } objStorage.deleteObjectsByKeys(bucket, keys); - deletedObjects = true; } - continuationToken = batch.isTruncated() - ? (restartListingAfterDelete(bucket) && deletedObjects - ? "" : batch.getContinuationToken()) - : null; + continuationToken = batch.isTruncated() ? batch.getContinuationToken() : null; } while (continuationToken != null); } - protected boolean restartListingAfterDelete(String bucket) { - return false; - } - /** Parses {@code uri} respecting the underlying client's path-style configuration. */ - protected ObjectStorageUri parseUri(String uri) { + private ObjectStorageUri parseUri(String uri) { return ObjectStorageUri.parse(uri, usePathStyle); } @@ -226,7 +217,7 @@ protected ObjectStorageUri parseUri(String uri) { * Computed by stripping the parsed key off the original URI string, so it is correct * for both virtual-hosted and path-style URIs without re-deriving the host. */ - protected static String uriBase(String uri, ObjectStorageUri parsed) { + private static String uriBase(String uri, ObjectStorageUri parsed) { String key = parsed.key(); String base = key.isEmpty() ? uri : uri.substring(0, uri.length() - key.length()); return base.endsWith("/") ? base : base + "/"; @@ -1112,7 +1103,7 @@ private static List compactPrefixes(List prefixes) { return compact; } - protected static int compareUtf8Binary(String left, String right) { + private static int compareUtf8Binary(String left, String right) { byte[] leftBytes = left.getBytes(StandardCharsets.UTF_8); byte[] rightBytes = right.getBytes(StandardCharsets.UTF_8); int commonLength = Math.min(leftBytes.length, rightBytes.length); @@ -1151,7 +1142,7 @@ private PrefixExpansion(List values, int nextIndex) { *
  • {@code data_{-1..1}.csv} → unchanged (no expansion)
    • * */ - protected static String expandNumericRanges(String pattern) { + private static String expandNumericRanges(String pattern) { java.util.regex.Pattern rangeSegment = java.util.regex.Pattern.compile( "(-?\\d+)\\.\\.(-?\\d+)"); java.util.regex.Pattern simpleRange = java.util.regex.Pattern.compile( diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index b7057564c56fc0..3a45208ea9bf11 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -392,17 +392,11 @@ enum TUpdateMode { OVERWRITE = 2 // insert overwrite } -enum TObjectStorageChecksumAlgorithm { - CRC32C = 0 -} - struct TS3MPUPendingUpload { 1: optional string bucket 2: optional string key 3: optional string upload_id 4: optional map etags - 5: optional TObjectStorageChecksumAlgorithm checksum_algorithm - 6: optional map part_checksums } struct THivePartitionUpdate { diff --git a/goal.md b/goal.md deleted file mode 100644 index 607715f7c3612b..00000000000000 --- a/goal.md +++ /dev/null @@ -1,1225 +0,0 @@ -# Apache Doris S3 Express One Zone 落地设计 - -## 1. 文档信息 - -| 项目 | 内容 | -| --- | --- | -| 状态 | Proposed | -| 最后核对日期 | 2026-07-16 | -| 实现基线 | take-over-63409，HEAD 0a193178c8 | -| 关联 PR | [apache/doris#63409](https://github.com/apache/doris/pull/63409) | -| 目标读者 | Doris BE、FE、Cloud、测试与发布维护者 | - -本文给出 Doris 支持 AWS S3 Express One Zone 的可实施方案。严格来说，S3 Express One Zone 是 storage class，Directory Bucket 是承载它的 bucket 类型；本文 P0 专指 Availability Zone 中使用 S3 Express One Zone 的 Directory Bucket。这里的“支持”不是简单地让一次 PutObject 成功，而是让 endpoint 解析、CreateSession、凭证刷新、checksum、multipart、ListObjectsV2、失败清理、普通 S3 兼容性和可观测性形成完整闭环。 - -本文中的 AWS 行为以 2026-07-16 的官方文档为准。S3 Directory Bucket 的能力仍在演进，开发和发布前必须重新核对文末的 AWS API 白名单与差异文档。 - -## 2. 结论先行 - -最终方案采用以下决策： - -1. 把 Directory Bucket 作为一种 S3 客户端能力模型，而不是通过“是否关闭 MD5”或 endpoint 子串做临时分支。 -2. Doris 只配置长期 IAM credentials 或既有 credentials provider。CreateSession、五分钟 session credentials 的缓存与刷新全部交给支持 S3 Express 的 AWS SDK，Doris 不自行实现 token manager。 -3. Directory Bucket 数据面不设置 endpointOverride，由 SDK 根据完整 bucket 名和 region 解析 Zonal endpoint；强制 HTTPS 和 virtual-hosted addressing，拒绝 path-style。 -4. Directory Bucket 写入统一显式使用 CRC32C。PutObject 发送对象 checksum；multipart 完整贯通 CreateMultipartUpload、UploadPart 和 CompleteMultipartUpload 的 part checksum。 -5. Directory Bucket 的 ListObjectsV2 不能沿用普通 S3 的 StartAfter 和字典序假设。Doris 使用 continuation token 扫描、客户端过滤和排序来保持现有 FileSystem 接口语义。 -6. 首个正式支持范围限定为 Doris 原生 S3 文件系统路径，包括 BE 读写以及与之配套的 FE 校验和列举。Cloud Storage Vault、Hadoop/S3A、预签名 URL、Access Point 不在首期支持声明中。 -7. 当前 PR 不能直接作为“Doris 已完整支持 S3 Express”合入。它已经验证了 BE 基础方向，但还缺少 FE、Cloud 边界、multipart 完整性、列举语义、失败 Abort、回归保护和真实 Directory Bucket 自动化测试。 - -## 3. 产品范围 - -### 3.1 首期 P0 支持矩阵 - -| Doris 使用面 | P0 状态 | 说明 | -| --- | --- | --- | -| BE 原生 S3 FileSystem exact-key 读、Head | 支持 | GetObject、HeadObject，由 C++ SDK 自动选择 Zonal endpoint 和 session auth | -| BE 小对象写入 | 支持 | PutObject，显式 CRC32C，不发送 Content-MD5 | -| BE 大对象写入 | 支持 | Create、UploadPart、Complete 全链路 CRC32C；失败时主动 Abort | -| BE 删除 | 支持 | DeleteObject；DeleteObjects 必须使用 flexible checksum | -| 原生 S3 前缀列举和 FE glob | 支持，但有代价 | 使用 continuation token 扫描相关目录；FE 客户端过滤、排序并应用 startAfter、maxFiles、maxBytes | -| FE S3 Resource 连通性检查 | 支持 | Java SDK v2，不强制通用 SigV4 signer，不 override Zonal endpoint | -| FE 新 S3 FileSystem 客户端 | 支持 | Java SDK v2，复用同一 credentials provider 和 bucket 分类规则 | -| 普通 AWS S3 | 保持现状 | endpoint、MD5、path-style 等既有行为不能回归 | -| MinIO、COS、OSS 等 S3-compatible | 保持现状 | 不因 bucket 名恰好以 --x-s3 结尾而切换到 AWS Express 模式 | - -### 3.2 P0 明确不支持 - -| 使用面 | 处理方式 | -| --- | --- | -| Cloud Storage Vault | FE 与 Meta Service 的创建、修改入口都 fail fast；原因见第 12 节 | -| 依赖 Hadoop S3A 的 External Catalog | 在确认所用 Hadoop/AWS SDK 版本支持 Directory Bucket 之前 fail fast | -| AWS Java SDK v1 路径 | 不支持；必须迁移到 Java SDK v2 后才能纳入 | -| Presigned URL | Directory Bucket 模式明确返回 NotSupported，完成独立鉴权和过期时间测试后再开放 | -| Directory Bucket Access Point | 不支持；其 host 和资源标识不同于直接 bucket | -| CopyObject、UploadPartCopy、RenameObject | 不作为 P0 对外承诺，实际调用面需要分别验证 | -| SSE-C、DSSE-KMS、对象 ACL、Object Lock、Requester Pays、对象 tag | Directory Bucket 不支持或不在 P0 | -| Doris 创建、删除 Directory Bucket | 不支持；bucket 由管理员预先创建 | -| Local Zone Directory Bucket | 不在 P0；P0 只声明 Availability Zone 中的 S3 Express One Zone | - -### 3.3 非目标 - -- 不通过 Doris 自研 HTTP 签名或 token 缓存替代 AWS SDK。 -- 不把 S3 Express 当成普通 S3 的透明性能开关。 -- 不承诺跨 AZ 的低延迟，也不在 Doris 内自动查询 IMDS 判断节点所在 AZ。 -- 不把单 AZ Directory Bucket 描述为普通 S3 的同等耐久性替代品。 -- 不在本次设计中改变 Doris 对象路径、文件格式或持久化元数据格式。 - -## 4. AWS 硬约束 - -### 4.1 Bucket、endpoint 与寻址 - -Availability Zone Directory Bucket 的完整名称为： - - ----x-s3 - -例如： - - analytics-hot--use1-az4--x-s3 - -zone-id 是全局一致的 Availability Zone ID，例如 use1-az4，不是账户内可能映射不同的 Availability Zone 名称。 - -数据面和控制面的 endpoint 不能混用： - -| 平面 | 典型操作 | endpoint | 寻址方式 | 认证 | -| --- | --- | --- | --- | --- | -| Zonal 数据面 | CreateSession、GetObject、PutObject、ListObjectsV2、multipart、Delete | s3express-..amazonaws.com | 只支持 virtual-hosted | 默认 session auth，由 SDK处理 | -| Regional 控制面 | 创建/删除 bucket、lifecycle、bucket policy 等 | s3express-control..amazonaws.com | path-style | 长期 IAM credentials 与 SigV4 | - -Doris P0 只需要对象数据面。客户端应把 bucket 和 region 交给 SDK endpoint rules，不能把用户填写的 endpoint 直接覆盖成 Zonal host，更不能把 s3express-control 当成对象 endpoint。 - -### 4.2 CreateSession - -- IAM principal 必须拥有目标 Directory Bucket 上的 s3express:CreateSession。 -- SDK 根据 --x-s3 bucket 名自动请求 bucket-scoped session credentials。 -- session credentials 有效期为五分钟；SDK 负责缓存、并发复用和临近过期刷新。 -- Doris 只保存或传递长期 credentials provider 所需的信息，不把 SDK 生成的 session access key、secret key 或 token 写入 FE、Meta Service、配置、日志或 RPC。 -- SDK 对少数使用长期 IAM 签名的操作有自己的签名选择逻辑，Doris 不应强制指定 AwsS3V4Signer 或自行指定 service name。 - -最小读写 IAM 示例： - - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": "s3express:CreateSession", - "Resource": "arn:aws:s3express:us-east-1:123456789012:bucket/analytics-hot--use1-az4--x-s3" - } - ] - } - -生产环境应使用角色和最小权限策略。若需要只读 session，应通过 AWS 支持的 s3express:SessionMode 条件约束 IAM，而不是让 Doris 修改 session token。 - -### 4.3 Checksum 与 ETag - -- Directory Bucket 的 PutObject 和 UploadPart 不支持 Content-MD5。 -- CRC32、CRC32C、SHA-1、SHA-256 可用于数据完整性校验；AWS 推荐高性能场景使用 CRC32 或 CRC32C。 -- 本设计固定选择 CRC32C，避免不同 SDK 版本的隐式默认值变化，并复用 Doris 已有的硬件优化 CRC32C 实现。 -- Directory Bucket 的 ETag 是不透明标识，不能假设为对象 MD5。 -- DeleteObjects 的 XML request body 必须带 Content-MD5 或 flexible checksum；Directory Bucket 路径使用 CRC32C。 - -### 4.4 Multipart - -- Directory Bucket 的 part number 必须从 1 开始连续递增；Complete 时不允许空洞。 -- part 可以并发上传，Doris 在 Complete 前按 part number 排序并验证连续性。 -- 一旦声明 multipart checksum algorithm，各 UploadPart 和 CompleteMultipartUpload 必须携带一致的 part checksum。 -- 未完成的 multipart 会持续占用存储并计费。应用必须主动 Complete 或 Abort，lifecycle 只能作为兜底。 - -### 4.5 ListObjectsV2 - -Directory Bucket 与普通 S3 的关键差异： - -- 返回结果不保证按 key 字典序排序。 -- 不支持 StartAfter。 -- Prefix 只支持以 / 结尾的值。 -- Delimiter 只支持 /。 -- 分页必须使用服务端返回的 opaque continuation token。 -- 列表中可能出现仅由未完成 multipart 产生的 prefix。 - -因此，任何依赖“服务端排序 + last key 续扫”的 Doris 代码都必须改造，不能只修改 endpoint 和鉴权。 - -### 4.6 能力限制 - -- 不支持 S3 Versioning。 -- 当前 AWS API 支持 Directory Bucket lifecycle，但只支持对象过期和取消未完成 multipart；不支持非当前版本过期、transition 和 tag filter。 -- P0 使用 bucket 默认 SSE-S3。SSE-KMS 需要单独验证 CreateSession、bucket 默认加密和请求 header 一致性后再开放。 -- StorageClass header 应省略或使用 EXPRESS_ONEZONE，不能发送 STANDARD。 -- Access Point 已是 AWS 独立能力，但不在 P0；不能误写成 AWS 永久不支持。 - -## 5. 当前 PR 评估 - -### 5.1 当前分支 - -take-over-63409 相对 upstream/master 的 merge-base e05c331da1 有三个提交： - -1. 4096f12a4f：PutObject 和 UploadPart 增加 CRC32C，并为 Express 关闭 Content-MD5。 -2. 98b7adffee：AWS SDK C++ 1.11.219 升级到 1.11.400，并改用 S3ClientConfiguration。 -3. 0a193178c8：集中和收紧 S3 Express 检测。 - -原 GitHub PR #63409 仍只有前两个提交；第三个检测修复尚未进入原 PR。原 PR 当前为 Changes requested。 - -### 5.2 已经正确的方向 - -- be/src/util/s3_util.cpp 开始使用 S3ClientConfiguration 和 S3EndpointProvider。 -- Directory Bucket 模式不再设置 endpointOverride。 -- disableS3ExpressAuth=false 时可由 SDK 接管 CreateSession。 -- be/src/io/fs/s3_obj_storage_client.cpp 已验证显式 CRC32C 的基本写入方向。 -- 普通 S3 仍保留 Content-MD5 分支。 -- common/cpp/aws_logger.h 已适配新 SDK 的 vaLog 接口。 - -### 5.3 必须修正的事实描述 - -AWS SDK C++ 1.11.219 并非完全不支持 S3 Express。AWS SDK C++ 维护者说明 S3 Express customizations 从 1.11.212 已合入；1.11.219 源码中已经存在 CreateSession、S3 Express signer/provider、disableS3ExpressAuth 和 endpoint rules。旧的 S3Client 构造函数在该版本也会创建 endpoint provider 和 Express signer provider。 - -因此： - -- 可以继续采用 1.11.400 作为 Doris 的固定版本，因为 AWS 建议采用包含后续 bug fix 的较新版本。 -- PR 描述不能把“1.11.219 没有 S3 Express 能力”作为根因。 -- 改成显式 S3ClientConfiguration 的价值是让配置和 endpoint provider 更清楚、可测试，而不是它单独开启了 Express。 -- 发布前要用 pinned SDK 的集成测试证明 endpoint、session refresh 和 checksum，而不是依赖版本号推断。 - -### 5.4 当前实现缺口和回归风险 - -| 问题 | 具体触发链 | 设计要求 | -| --- | --- | --- | -| 覆盖面只有 BE 基础路径 | FE 和 Cloud 有独立 S3 client builder | 分别改造或明确排除，不能宣称 Doris 全面支持 | -| 检测仍把 bucket/endpoint 字符串作为布尔值 | 第三方 bucket 恰好以 --x-s3 结尾会被跳过自定义 endpoint | 引入 service-aware capability resolver，联合 bucket、官方 endpoint authority 和 override mode | -| disableS3ExpressAuth 与自研检测绑定 | AWS 普通 bucket 被强制关闭 SDK 默认能力，第三方行为与 endpoint 子串耦合 | 官方 AWS service 保持 SDK 默认 false；自定义 endpoint client 明确禁用 | -| 新配置从默认值重建 | be/src/util/s3_util.cpp 直接默认构造 S3ClientConfiguration | 从现有 ClientConfiguration 转换，保留 CA、timeout、retry 等语义 | -| payload signing 策略变化 | 旧构造显式 Never，新构造使用默认 RequestDependent | 普通路径显式保持现状，并做回归与性能测试 | -| s3_disable_content_md5 是动态全局配置 | 配置只在 cached S3ObjStorageClient 构造时快照 | P0 删除该用户开关；checksum 由 bucket capability 决定 | -| cache key 不完整 | S3ClientConf::get_hash() 缺 need_override_endpoint 和新策略 | 所有影响 client 行为的字段进入 key | -| multipart checksum 只有 UploadPart | Create 未声明算法、Complete 只传 ETag | 完整传播 CRC32C | -| 写失败不 Abort | S3FileWriter::_abort 仅声明，底层无接口 | 增加 abort API 并覆盖取消、失败和并发上传收敛 | -| DeleteObjects 未覆盖 | Directory Bucket request body 不接受既有 MD5 假设 | 显式 CRC32C 或验证 SDK 注入，必须有 request capture test | -| List 仍依赖普通 S3 语义 | FE glob 直接下推 StartAfter，Prefix 可不以 / 结尾 | continuation token + 本地过滤、排序 | -| 可能泄露 token | S3ClientConf::to_string() 直接输出 token | token 和 externalId 必须遮蔽或完全移除 | -| 默认 HTTP 风险 | s3_client_http_scheme 和 S3Resource 历史路径可生成 HTTP | Directory Bucket 强制 HTTPS | -| 测试仅覆盖检测 | 当前新增测试只有少量 is_s3_express case | 增加 client、request、multipart、list、refresh 和真实 AWS 测试 | - -### 5.5 Doris code-review 关键检查点结论 - -| 检查点 | 当前 PR 结论 | -| --- | --- | -| 目标是否完成、是否有测试证明 | 仅完成 BE PutObject/UploadPart 的基础尝试；没有证明完整 Doris 调用链，也没有 session refresh、multipart、list 或真实 AWS 自动化测试 | -| 修改是否最小清晰 | 差异本身较小，但用全局 MD5 开关和 bool Express 状态承载多种能力，抽象不足 | -| 并发 | S3FileWriter 并发上传 part；当前没有等待所有 part 后 Abort 的失败收敛设计。SDK session provider 的并发复用也未测试 | -| 生命周期 | 动态配置在 cached client 构造时快照；client、五分钟 session、multipart upload 的创建与销毁生命周期尚未闭环 | -| 新增配置 | s3_disable_content_md5 声明为 mutable，但运行时更新对已有 client 不生效，不能按当前形态保留 | -| 兼容与滚动升级 | 没有 wire/storage format 变化，但重建 S3ClientConfiguration 可能改变 timeout 和 payload signing；旧 BE 不能用于 Directory Bucket | -| 平行调用路径 | FE 两套 Java builder、Hive BE-upload/FE-complete、Cloud recycler/client、特殊 BE client 创建点没有同步覆盖 | -| 特殊条件判断 | bucket/endpoint 字符串判断不能区分官方 AWS 与自定义 S3-compatible endpoint | -| 测试覆盖 | 新增测试只覆盖少量识别输入；请求字段、错误路径、负例和端到端覆盖不足 | -| 可观测性 | 没有 Directory Bucket 专用 metrics；现有 S3ClientConf::to_string() 还会输出明文 token | -| 持久化与协议 | bucket capability 无需持久化；但 BE 上传、FE 完成的 Hive multipart 必须增加可选 Thrift checksum 字段 | -| 数据写入正确性 | 小对象 CRC32C 方向正确，但 multipart checksum、DeleteObjects checksum 和失败 Abort 不完整 | -| 性能 | client/session 复用方向正确；需验证 payload signing、CRC32C CPU 和 Directory Bucket 全量 list 的成本 | - -## 6. 统一能力模型 - -### 6.1 C++ 数据结构 - -在 BE 和 Cloud 都可依赖的 common/cpp 层定义纯数据类型和无网络副作用的 resolver。名称可按现有构建组织调整，建议语义如下： - - enum class S3BucketType { - GENERAL_PURPOSE, - DIRECTORY - }; - - enum class S3EndpointMode { - EXPLICIT_OVERRIDE, - AWS_SDK_RULES - }; - - enum class S3ChecksumPolicy { - CONTENT_MD5, - CRC32C - }; - - struct S3BucketCapabilities { - S3BucketType bucket_type; - S3EndpointMode endpoint_mode; - S3ChecksumPolicy checksum_policy; - bool require_https; - bool require_virtual_addressing; - bool supports_start_after; - bool list_is_lexicographic; - bool supports_versioning; - bool supports_presign; - }; - -resolver 输入必须包含： - -- storage provider（若当前 surface 有该信息） -- 完整 bucket 名 -- endpoint -- region -- use_path_style 或 use_virtual_addressing -- need_override_endpoint 或等价的“官方 AWS 服务/自定义 S3-compatible 服务”信息 -- 当前调用面是否声明支持 Directory Bucket - -resolver 输出不可只返回 bool。调用者应基于 capability 做 endpoint、auth、checksum、list 和功能门禁，避免散落多个 is_s3_express 判断。 - -### 6.2 判定真值表 - -| 服务意图 | Bucket | Endpoint/override | 结果 | -| --- | --- | --- | --- | -| AWS 服务 | 完整 --x-s3 后缀 | 空、标准 AWS regional endpoint 或匹配的官方 Zonal endpoint | DIRECTORY | -| AWS 服务 | 非 --x-s3 | 官方 s3express Zonal/control endpoint | 配置错误，fail fast | -| 自定义 S3-compatible 服务 | 任意名称，包括 --x-s3 后缀 | 自定义 endpoint，明确需要 override | GENERAL_PURPOSE，不自动切 Express | -| 非 S3 provider | 任意名称 | 官方 s3express endpoint | 配置错误，fail fast | - -这里的 AWS 服务不能只由用户属性 provider 或 C++ 的 ObjStorageType::AWS 判断。FE 合法的通用 S3 provider 值是 S3，进入 C++ 后又可能映射成 ObjStorageType::AWS；MinIO 等 S3-compatible 配置会走同一类枚举。resolver 必须结合 need_override_endpoint 和 endpoint authority：明确使用自定义 endpoint override 的客户端按 GENERAL_PURPOSE 处理；只有指向官方 AWS 服务且 bucket 后缀合法时才进入 DIRECTORY。endpoint 仍然不能单独作为 Directory Bucket 的判定依据，不能因为字符串中出现 s3express 就启用 session auth。 - -官方 endpoint 的识别应复用 AWS partition/endpoint metadata 或集中维护的 parser，不能把 .amazonaws.com 写死后遗漏其他 partition，也不能维护会快速过期的 region/AZ allowlist。 - -### 6.3 Bucket 解析 - -- 只接受完整后缀 --x-s3。 -- 从最后两个双连字符段解析 zone-id，不维护静态 AZ allowlist。 -- region 必填。 -- 若 endpoint 同时包含 region 或 zone-id，发现明显不一致时返回包含 bucket、region、endpoint 的可操作错误。 -- 不仅检查 endpoint 字符串；最终 endpoint 仍由 SDK rules 生成。 - -### 6.4 跨语言一致性 - -FE Java 无法直接复用 C++ 类型，因此应： - -1. 在 S3URI/S3Properties/S3FileSystemProperties 附近实现同样的 service-aware resolver。 -2. 增加一份跨语言测试向量，覆盖 bucket、provider（若有）、endpoint、override mode、region、path-style 和预期结果。 -3. C++ 和 Java 单测读取同一组向量，防止后续规则漂移。 - -P0 不为 bucket type 新增 protobuf、Thrift 或持久化字段。各 surface 使用已有 bucket、region、endpoint、path-style，并结合本地 endpoint override 规则确定性派生 capability；不能假设 provider 已在所有路径传递，因为 S3FileSystemProperties 和 S3Properties.getS3TStorageParam() 当前不会完整传递它。 - -multipart checksum 是另一件事：BE 上传、FE 完成的 Hive 写入必须新增可选 Thrift 字段，详见第 8.5 节。可选字段保证旧消息可解析，但不代表混合版本可以启用 Directory Bucket。 - -### 6.5 FE 的两级解析 - -S3FileSystemProperties.bucket 在部分调用面可以为空，实际 bucket 来自每次 remotePath；S3ObjStorage 当前也只维护一个 lazy S3Client。因此 FE 不能在 buildClient 时把整个 client 固化成某个 bucket type。 - -FE 使用两级模型： - -1. service-level config 在创建 S3ObjStorage 时确定： - - AWS_SDK_ROUTED：endpoint 为空或为可安全归一化的标准 regional endpoint；允许 Directory Bucket。 - - AWS_EXPLICIT_ENDPOINT：FIPS、dualstack、accelerate 或其他必须保留的显式 AWS endpoint；P0 只允许普通 bucket。 - - CUSTOM_S3_COMPATIBLE：保留 endpointOverride，禁用 Express session auth。 -2. request-level capability 在每个方法解析 remotePath 后，根据实际 bucket、service-level config、region 和 path-style 派生： - - AWS_SDK_ROUTED + --x-s3 为 DIRECTORY。 - - AWS_SDK_ROUTED + 普通 bucket 为 GENERAL_PURPOSE。 - - AWS_EXPLICIT_ENDPOINT + 普通 bucket 为 GENERAL_PURPOSE；遇到 --x-s3 时 fail fast。 - - CUSTOM_S3_COMPATIBLE 下所有 bucket 都按 GENERAL_PURPOSE。 - -为保证普通 AWS S3 不回归，S3ObjStorage 将当前单一 lazy client 改成 clientFor(bucket)： - -- general client 完整保留已有 endpointOverride、FIPS/dualstack、path-style 和兼容配置。 -- directory client 只在 AWS_SDK_ROUTED 的 Directory Bucket 请求出现时惰性创建，固定为无 endpointOverride、HTTPS、virtual-hosted、Express auth enabled。 -- 两个 client 复用同一长期 credentials provider，但生命周期和配置缓存分开。 -- 一个 directory client 可以服务同 region 的多个 Directory Bucket，SDK 的 Express identity cache 按 bucket 和 credentials identity 隔离 session。 - -Put、multipart、List、Copy、presign 等方法必须使用当前 URI 的 request-level capability 和 clientFor(bucket)，不能把第一次请求的 bucket type 缓存在 S3ObjStorage 上，也不能为每个对象创建新 client。 - -## 7. 客户端与请求数据流 - - FE properties / URI / storage resource - | - v - service-aware capability resolver - | - +---- unsupported surface: fail fast - | - v - existing PB / Thrift storage parameters - only long-term credentials configuration - | - v - BE S3ClientFactory client cache - | - v - AWS SDK endpoint rules + credentials provider - | - +---- CreateSession - | | - | +---- SDK-only five-minute session credentials - | - v - bucket-specific Zonal data endpoint - | - +---- Get / Head / List - +---- Put / Multipart / Delete - -Hive multipart 的事务提交链路另有一条 BE 到 FE 的 TS3MPUPendingUpload 消息，只传算法、ETag 和 part checksum 等完成上传所需元数据，仍然不传 session credentials。 - -关键生命周期： - -- client 以 bucket、region、service kind、endpoint mode、credentials provider identity 等为 key 长期缓存。 -- 首次 Directory Bucket 请求触发 SDK 获取 session。 -- 并发请求复用同一个 client 和 session provider。 -- SDK 在 session 过期前刷新；Doris 不观察和持久化 token。 -- client 销毁时 session credentials 只随进程内对象释放。 - -## 8. BE 详细改造 - -### 8.1 S3ClientConf 与 cache key - -文件： - -- be/src/util/s3_util.h -- be/src/util/s3_util.cpp - -改造： - -1. 在 S3ClientConf 初始化完成且 bucket 已知后调用 capability resolver。 -2. get_hash() 至少加入 need_override_endpoint、bucket type、endpoint mode、checksum policy、scheme、addressing style。 -3. 不继续用简单 XOR 拼接易交换字段的 hash；按字段顺序使用 hash combine。 -4. credentials provider 类型、role ARN、external ID、session token identity 必须继续参与隔离。 -5. to_string() 只能输出遮蔽后的 access key；token、secret key 和 external ID 不输出明文。 -6. client cache 不跨不同 bucket 复用 Directory Bucket client，确保 session credentials 的 bucket scope 不被混用。 - -### 8.2 S3ClientFactory - -文件： - -- be/src/util/s3_util.cpp - -普通 S3 路径： - -- 保留当前 endpointOverride 规则。 -- 保留 useVirtualAddressing、timeout、CA、retry 和 payload signing 的原有语义。 -- S3-compatible provider 不因名称命中 --x-s3 而改变行为。 - -Directory Bucket 路径： - -- 从 S3ClientFactory::getClientConfiguration() 返回的现有公共 ClientConfiguration 通过 converting constructor 构造 S3ClientConfiguration，避免丢失 aws_client_request_timeout_ms 等公共默认值。 -- region 使用用户配置。 -- scheme 强制 HTTPS；若配置显式要求 HTTP，直接返回配置错误。 -- useVirtualAddressing 强制 true；若用户配置 path-style，直接返回配置错误。 -- endpointOverride 保持空。 -- disableS3ExpressAuth 保持 false。 -- 使用 S3EndpointProvider。 -- credentials provider 仍是 Doris 已有 static、default chain、web identity、instance profile 或 assume-role provider。 -- 对普通路径显式保留 PayloadSigningPolicy::Never；Directory Bucket 路径让 SDK 的 Express customization 决定必要的签名行为，并用性能测试验证没有重复 payload hash。 - -不要把 disableS3ExpressAuth 设置成 !is_directory。建议规则是： - -- 官方 AWS 服务：false，允许 SDK 根据实际 bucket 选择。 -- 自定义 S3-compatible endpoint：true，防止 AWS SDK 根据特殊 bucket 名误启用 Express。 - -### 8.3 特殊构造路径审计 - -所有在 bucket 未知时创建 S3ClientConf 的路径必须修正。例如 be/src/exprs/function/ai/embed.h 应先解析 URI 并写入 bucket，再创建 client。审计要求： - - rg "S3ClientConf|create_s3_client|S3ClientFactory" be cloud - -每个直接构造 client 的路径都必须归入以下之一： - -- 使用统一 factory 和 capability resolver。 -- 明确仅支持普通 S3，并在 Directory Bucket 输入时 fail fast。 -- 有独立的 Directory Bucket 实现和测试。 - -### 8.4 PutObject - -文件： - -- be/src/io/fs/s3_obj_storage_client.h -- be/src/io/fs/s3_obj_storage_client.cpp - -改造： - -- S3ObjStorageClient 持有 S3BucketCapabilities，而不是 is_s3_express 和 _disable_content_md5 两个布尔值。 -- DIRECTORY 时： - - SetChecksumAlgorithm(CRC32C) - - 计算 Base64 编码的 CRC32C 并 SetChecksumCRC32C - - 不 SetContentMD5 -- GENERAL_PURPOSE 时保持当前 Content-MD5 行为。 -- 对空对象、非对齐 buffer、最大单次 PutObject 边界增加已知向量测试。 -- 不把 ETag 与本地 MD5 比较。 - -当前 s3_disable_content_md5 全局动态配置不进入最终 P0。若未来确实需要为第三方存储配置 checksum，应设计为每个 resource 的 AUTO、MD5、CRC32C、NONE 枚举，并贯穿 FE、PB/Thrift、BE 及 cache key，不能复用当前全局布尔值。 - -### 8.5 Multipart 完整校验 - -相关文件： - -- be/src/io/fs/obj_storage_client.h -- be/src/io/fs/s3_obj_storage_client.cpp -- be/src/io/fs/s3_file_writer.cpp -- be/src/io/fs/s3_file_writer.h - -Directory Bucket 流程： - -1. CreateMultipartUpload 设置 ChecksumAlgorithm=CRC32C。 -2. UploadPart 为每个 part 计算并发送 CRC32C。 -3. 校验服务端返回的 ChecksumCRC32C 与本地值一致；缺失或不一致均返回失败。 -4. ObjectStorageUploadResponse 增加可选 checksum_crc32c。 -5. ObjectCompleteMultiPart 增加可选 checksum_crc32c。 -6. S3FileWriter 保存 part number、ETag、CRC32C。 -7. CompleteMultipartUpload 的每个 CompletedPart 同时设置 ETag、PartNumber、ChecksumCRC32C。 -8. Complete 前排序并断言 part number 为 1..N 连续序列；违反不变量时失败，不能跳过空洞继续提交。 - -普通 S3FileWriter 在同一 BE 内完成上传时，上述内部结构足够；但 Hive 写入还存在“BE 上传 part，FE 提交事务后 Complete”的跨进程链路： - -- gensrc/thrift/DataSinks.thrift 的 TS3MPUPendingUpload 当前只有 bucket、key、upload_id 和 etags。 -- be/src/exec/sink/writer/vhive_partition_writer.cpp 只把 part ETag 写入 Thrift。 -- fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java 在 FE 发起 Complete。 -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java 当前把 ETag map 转成 UploadPartResult。 - -P0 必须给 TS3MPUPendingUpload 增加可选字段： - - 5: optional TObjectStorageChecksumAlgorithm checksum_algorithm - 6: optional map part_checksums - -其中 P0 枚举至少包含 CRC32C，part_checksums 的 key 与 etags 的 part number 一致。对应改造： - -1. VHivePartitionWriter 从 S3FileWriter.completed_parts() 同时填入算法和每个 part 的 CRC32C。 -2. HMSTransaction 把两个新字段交给 ObjFileSystem。 -3. ObjFileSystem 构造带 checksum 的 UploadPartResult。 -4. S3ObjStorage 在 Complete 时为每个 CompletedPart 设置 CRC32C。 -5. Directory Bucket 模式若新字段缺失，FE 必须拒绝 Complete 并走 Abort；不能降级成只传 ETag。 -6. 普通 S3 和旧消息可以不设置可选字段。 - -这些改动不改变持久化对象格式，但改变了可选 Thrift 消息。新代码可解析旧消息；语义上仍不允许混合版本启用 Directory Bucket，因为旧 FE 会忽略新字段，旧 BE 也不会生成它们。Azure、OSS、COS、OBS 等实现不使用 checksum 字段，但必须继续兼容现有构造。 - -若 pinned C++ SDK 的 request model 对 composite checksum 有额外约束，以 SDK 1.11.400 的实际模型和真实 AWS 测试为准；不能退化成“只要 UploadPart 成功就视为完整支持”。 - -### 8.6 AbortMultipartUpload - -当前 S3FileWriter::_abort 只有声明。P0 必须： - -1. 给 ObjStorageClient 增加 abort_multipart_upload。 -2. S3 实现调用 AWS AbortMultipartUpload；NoSuchUpload 视为幂等成功。 -3. 等待所有 in-flight UploadPart future 收敛后再 Abort，避免 part 上传与 Abort 竞态。 -4. Create 成功后的任一上传、Complete、取消或 close 失败都进入 Abort。 -5. 若业务操作失败且 Abort 也失败，返回保留原始错误并附带 cleanup 错误的 Status。 -6. 析构函数不执行可能阻塞或抛异常的网络请求，只记录未正常 close 的 metric 和 warning。 -7. 服务端 AbortIncompleteMultipartUpload lifecycle 作为最终兜底，不替代应用主动清理。 - -### 8.7 DeleteObjects - -审计 BE 和 Cloud 的批量删除实现： - -- Directory Bucket 使用 CRC32C 校验 XML request body。 -- 不设置 Content-MD5。 -- request capture 单测必须断言 algorithm 和 checksum header。 -- 保留每个 object 的失败明细；部分失败不能静默当作成功。 - -### 8.8 读取 - -- GetObject、HeadObject 由 SDK 处理 endpoint 和 session auth。 -- P0 不改变 Doris range read 语义。 -- 若启用 ChecksumMode，必须验证 ranged GET 的 AWS 行为；不能要求服务端对任意 range 返回完整对象 checksum。 -- TLS 是读取传输保护的最低要求。 -- 所有使用 ETag 做缓存 identity 的代码只把它当不透明字符串。 - -## 9. ListObjectsV2 兼容方案 - -### 9.1 现有冲突 - -fe-filesystem 的 FileSystem 接口把 startAfter 定义为字典序游标；S3CompatibleFileSystem.globListWithLimit 当前把它直接下推给 ListObjectsV2，并按服务返回顺序计算 maxFiles、maxBytes 和 GlobListing.maxFile。这在 Directory Bucket 上不成立。 - -相关文件： - -- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/FileSystem.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectListOptions.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java - -仅把 prefix 扩展到父目录还不够，因为： - -- StartAfter 请求会被拒绝。 -- 返回顺序不稳定，提前应用 maxFiles 或 maxBytes 会漏掉字典序更小的 key，并产生错误的 maxFile 游标。 -- 原始最长非通配 prefix 可能不以 / 结尾。 - -### 9.2 P0 算法 - -输入： - -- originalPrefix：Doris 语义中的最长前缀 -- directoryPrefix：originalPrefix 截断到最后一个 /，没有 / 时为空 -- globPattern -- startAfter -- maxFiles -- maxBytes - -执行： - -1. 只向 AWS 发送以 / 结尾的 directoryPrefix。 -2. 不发送 StartAfter。 -3. 用 NextContinuationToken 读取到 IsTruncated=false。 -4. 对每一页在客户端过滤 originalPrefix 和 globPattern。 -5. 在客户端应用 key > startAfter。 -6. 收集匹配对象的 key、size 和 modification time，并按 Doris 既有 key comparator 排序。 -7. 按排序结果依次加入 FileEntry；加入当前对象后，如果 files.size 达到 maxFiles 或 totalSize 达到 maxBytes，则结束当前返回页。这与现有“包含触发阈值的对象”语义一致。 -8. GlobListing.maxFile 设置为返回页后的第一个匹配 key；若不存在下一项，则保持现有契约，使用最后一个已返回 key。 -9. maxFiles 有值且 maxBytes 未启用时，可以用大小为 maxFiles+1 的最大堆优化内存；只要 maxBytes 启用，阈值依赖排序后对象大小，最坏情况必须收集全部匹配项后排序，不能套用单一 limit 堆算法。 -10. 多个服务 prefix 的结果需要去重后再统一排序；Directory Bucket 路径通常只使用扩大的一个 directoryPrefix。 -11. 全程响应查询取消；超过现有内存限制时返回 ResourceLimit 错误，不能静默截断。 - -复杂度： - -- 服务请求数与 directoryPrefix 下的总对象数相关。 -- 只有 maxFiles 限制时可把内存优化为 O(maxFiles)，CPU 为 O(N log maxFiles)。 -- maxBytes 启用或没有限制时，最坏内存为 O(M)，排序 CPU 为 O(M log M)，M 是匹配对象数。 - -这是保持现有 FileSystem 契约的正确性成本。文档应建议用户为高频列举使用稳定、以 / 结尾且选择性高的目录前缀。 - -### 9.3 BE 与 Cloud 的不同契约 - -BE S3ObjStorageClient.list_objects 返回完整 FileInfo 集合，没有 FE GlobListing 的 maxFiles、maxBytes、maxFile 契约。BE 只需要： - -- continuation token 翻完所有服务页。 -- 任意非目录 prefix 先扩大到父目录，再客户端精确过滤。 -- 不承诺或依赖服务返回顺序；只有上层契约要求时才排序。 -- 未完成 multipart 产生的 CommonPrefixes 不当成已提交对象。 - -BE 和 Cloud 的递归删除也不能直接复用 FE 排序算法。尤其不能一边按 opaque continuation token 翻页，一边删除已经列出的对象后继续使用原 token。Directory Bucket 的安全算法是： - -1. 从空 continuation token 开始扫描，客户端过滤目标 prefix。 -2. 收集最多一个 DeleteObjects batch 的匹配 key；如果前几页没有匹配项，继续翻页，不能提前结束。 -3. 删除该 batch 后丢弃 continuation token，从头重新扫描。 -4. 重复直到一次完整扫描找不到匹配对象。 -5. 每个 DeleteObjects request 使用 CRC32C，并检查部分失败。 - -Cloud recycler 的 list_prefix、递归删除和 checker 只需要集合完整性和取消/重试语义，不需要 FE 的字典序分页结果。三条调用链共享 AWS 协议约束和 prefix 过滤 helper，但各自保留自己的返回契约。这里的 Cloud 算法只描述第 12.2 节的独立 P1：P0 不让 Directory Bucket 请求进入 Cloud recycler，必须在 Vault 持久化前拒绝。 - -## 10. FE 详细改造 - -### 10.1 S3URI 与 S3Properties - -相关文件： - -- fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java -- fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java - -现有 S3URI 已有 directory bucket 辅助逻辑，可以保留 bucket suffix 解析，但必须升级为 service-aware capability resolver。校验时： - -- OFFICIAL_AWS service + --x-s3 才进入 Directory Bucket。 -- region 必填。 -- path-style=false。 -- endpoint 必须是 HTTPS。 -- 推荐 endpoint 配置为标准 AWS regional endpoint，例如 https://s3.us-east-1.amazonaws.com；它只作为 region/配置提示，不下传为 Directory Bucket 的 endpoint override。 -- 对 control endpoint、region/zone 冲突给出明确错误。 - -若某个 surface 在校验阶段拿不到 bucket，不能只看 endpoint 猜测，也不能要求 S3FileSystemProperties.bucket 必填。先区分 AWS_SDK_ROUTED、AWS_EXPLICIT_ENDPOINT、CUSTOM_S3_COMPATIBLE，再在每次 remotePath 解析出 bucket 后确定 request-level capability。 - -### 10.2 fe-filesystem-s3 - -相关文件： - -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java -- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjStorage.java - -当前代码注释说“仅为非 AWS endpoint 设置 endpointOverride”，但实现对任何非空 endpoint 都 override。不能为支持 Directory Bucket 而一律删除官方 AWS endpointOverride，因为普通 bucket 可能依赖 FIPS、dualstack 或显式 regional endpoint。改造为： - -- general client： - - 完整保留当前 endpointOverride、path-style、FIPS/dualstack 和兼容 signer 行为。 - - CUSTOM_S3_COMPATIBLE 调用 disableS3ExpressSessionAuth(true)。 -- directory client： - - 只允许 AWS_SDK_ROUTED。 - - 不调用 endpointOverride。 - - region 必填。 - - disableS3ExpressSessionAuth(false)。 - - 不设置自定义 signer。 - - 使用 buildCredentialsProvider() 返回的同一 provider。 - - 强制 HTTPS 和 virtual-hosted，让 Java SDK v2 解析 Zonal endpoint/auth。 -- AWS_EXPLICIT_ENDPOINT 遇到 Directory Bucket 时 fail fast；P0 不静默丢弃用户指定的 FIPS/dualstack 语义。 - -每次请求解析 bucket 后由 clientFor(bucket) 选择 general 或 directory client。若当前 remotePath bucket 是 DIRECTORY 且 usePathStyle=true，在发送请求前 fail fast；普通 bucket 保留既有 path-style 行为。 - -S3ObjStorage 自身也执行 S3Resource ping 的 Put 和 multipart，不能只修 client builder： - -- 每个公开方法解析 S3Uri 后调用 capabilitiesFor(uri.bucket())，不缓存第一次请求的 bucket type。 -- Directory PutObject 显式选择 CRC32C，让 Java SDK 计算或发送 checksum，并用 request interceptor 测试实际 header。 -- Directory CreateMultipartUpload 声明 CRC32C。 -- Directory UploadPart 返回并保存 ChecksumCRC32C。 -- Java SPI 的 UploadPartResult 增加可选 checksum 字段，并保留现有两参数构造以避免影响 OSS、COS、OBS、Azure。 -- CompleteMultipartUpload 为每个 CompletedPart 传递 CRC32C。 -- CopyObject、presign 等未纳入 P0 的操作在 Directory Bucket 模式 fail fast。 -- 普通 S3-compatible 请求保持现有行为。 - -general/directory 两个 lazy client 各自使用线程安全的初始化方式。测试要覆盖并发首次请求，确认每种配置只创建一个 client，且两种配置不会串用。 - -### 10.3 fe-core S3Util - -相关文件： - -- fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java - -该文件使用 AWS Java SDK v2，但当前 builder： - -- 无条件 endpointOverride。 -- 强制 AwsS3V4Signer。 -- 可开启 path-style。 - -把 builder 收敛为接收规范化的 service-level config，并由已知 bucket 选择配置： - -- Directory Bucket 使用 AWS_SDK_ROUTED：不 override endpoint、不强制 signer，并启用 Express session auth。 -- 普通 AWS 或自定义 S3-compatible 保留现有显式 endpoint、signer 和 chunkedEncoding 兼容设置；自定义 endpoint 禁用 Express session auth。 -- AWS_EXPLICIT_ENDPOINT + Directory Bucket 在构建前 fail fast。 -- HeadBucket 连通性检查必须使用完整 bucket，并在调用前用 request-level capability 拒绝 Directory Bucket 的 path-style/HTTP 配置。 - -不建议继续扩散多个 buildS3Client overload。先收敛到一个接收规范化配置对象的内部 builder，再让旧 overload 委托它。 - -### 10.4 S3Resource - -相关文件： - -- fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java - -改造： - -- Directory Bucket 不允许缺省成 http://。 -- ping test 的 ListObjects prefix 必须使用父目录 slash prefix，并在客户端过滤目标 key。 -- ping test 复用现有 ObjStorage.abortMultipartUpload API；任何失败路径都 Abort，并把 Abort 失败附加到原始错误，不能像当前实现一样吞掉 cleanup 异常。 -- 错误信息区分 InvalidDirectoryBucketConfiguration、CreateSessionAccessDenied、UnsupportedS3Surface。 - -### 10.5 Presigned URL - -S3ObjStorage.getPresignedUrl 当前使用 static basic credentials 创建 presigner，和实际 client 的 default chain、session token、assume-role provider 不一致。P0 处理： - -- Directory Bucket 直接返回明确 NotSupported。 -- 普通 S3 保持现状。 -- 后续开放前必须让 presigner 使用统一 credentials provider，并验证 URL 的签名类型、bucket host、session token、最大有效期和 SDK 刷新行为。 - -### 10.6 Hadoop/S3A 和 Java v1 - -对所有 External Catalog、Hadoop FileSystem、AWS Java SDK v1 client 做调用面清单。这份清单是 P0 的合入产物和阻断门禁，至少逐项记录：用户入口、最终 I/O 实现及依赖版本、何时能取得完整 bucket、最早可确定拒绝的位置、负责人和自动化测试。初始清单至少覆盖： - -| 调用面家族 | 已知入口示例 | P0 处理 | -| --- | --- | --- | -| Doris 原生 S3 FileSystem | S3ObjStorage、S3FileSystem、S3Util | 按本设计适配并支持 | -| Hive BE-upload/FE-complete | VHivePartitionWriter、HMSTransaction、ObjFileSystem | 按第 8.5 节传播 multipart checksum 后支持 | -| Hudi Hadoop FileSystem | HudiScanNode 及其 Hadoop 配置构造 | 未证明所装载 S3A 版本支持前拒绝 | -| LakeSoul Hadoop FileSystem | LakeSoulUtils 及其 Hadoop 配置构造 | 未证明所装载 S3A 版本支持前拒绝 | -| Paimon/Hadoop 配置 | AbstractPaimonProperties 及对应 catalog/scanner | 未证明最终 I/O client 支持前拒绝 | -| be-java-extensions 与其他外部 catalog/scanner | 各扩展自己的 classloader 和 AWS/Hadoop 依赖 | 逐模块锁定实际依赖版本；未验证项拒绝 | -| AWS Java SDK v1 直接客户端 | fe-core 及扩展中引入 aws-java-sdk-s3 的路径 | P0 不支持，迁移或隔离到 Java SDK v2 后再开放 | - -P0 规则： - -- 若最终 I/O 由已适配的 Doris S3 FileSystem 执行，允许。 -- 若最终 I/O 由未经验证的 S3A 或 Java v1 client 执行，在能够取得完整 bucket 的最早确定点拒绝 Directory Bucket URI。bucket 在分析期可知时就在分析期拒绝；只能在运行时 URI 中取得时，必须在 filesystem/scanner factory 发出首个网络请求前拒绝。 -- 错误必须指出该 surface 尚未支持，不能在运行时表现为 403、错误 endpoint 或签名失败。 -- 每一行清单都必须有正向普通 S3 用例和 Directory Bucket 负例；清单未穷举或缺少测试时，PR 3 不满足完成条件。 - -## 11. 配置与用户使用 - -### 11.1 推荐配置 - -Directory Bucket 已由管理员在 us-east-1 的 use1-az4 创建： - - analytics-hot--use1-az4--x-s3 - -Doris S3 Resource 示例： - - CREATE RESOURCE "s3_express_hot" - PROPERTIES - ( - "type" = "s3", - "provider" = "S3", - "AWS_ENDPOINT" = "https://s3.us-east-1.amazonaws.com", - "AWS_REGION" = "us-east-1", - "AWS_BUCKET" = "analytics-hot--use1-az4--x-s3", - "AWS_ROOT_PATH" = "hot-data/", - "AWS_ACCESS_KEY" = "", - "AWS_SECRET_KEY" = "", - "use_path_style" = "false" - ); - -对象 URI： - - s3://analytics-hot--use1-az4--x-s3/hot-data/example.parquet - -优先使用 instance profile、web identity 或 assume role，避免在 Resource 中保存静态 AK/SK。上例只用于展示已有属性形状。 - -Doris 当前 S3 属性的合法 provider 值是 S3，不是 AWS。是否连接官方 AWS 服务由 endpoint authority 和 override mode 判断；不要为了本功能新增一个只在部分链路存在的 AWS 枚举。 - -### 11.2 endpoint 规则 - -- 用户不需要手工配置 bucket-specific Zonal endpoint。 -- 如果当前 Doris surface 强制要求 AWS_ENDPOINT，填写标准 regional S3 HTTPS endpoint。 -- Directory Bucket client 不把该值传给 endpointOverride。 -- 普通 bucket 继续保留显式 regional、FIPS、dualstack 和 S3-compatible endpoint 行为；P0 的 Directory Bucket 只接受 AWS_SDK_ROUTED，遇到 FIPS、dualstack 或其他显式 override 配置时拒绝，而不是静默改写。 -- s3express-control endpoint 不能用于对象 I/O。 -- 自定义 proxy 若会改写 Host、DNS 或 TLS SNI，不在支持范围。 - -### 11.3 部署要求 - -- 计算节点和 bucket 位于同一 AZ 才能获得设计目标中的低延迟和更低网络成本。 -- 跨 AZ 功能上可能可达，但不作为性能目标，且会增加成本和故障域复杂度。 -- 私网访问使用 com.amazonaws..s3express 的 gateway VPC endpoint；普通 S3 endpoint 或 PrivateLink 配置不能直接替代。 -- DNS、TLS SNI 和 Host header 必须保留 SDK 生成的 bucket-specific host。 - -## 12. Cloud Storage Vault 边界 - -### 12.1 为什么 P0 不支持 - -Cloud Storage Vault 是 Doris Cloud 的主持久化存储，不是临时 staging 目录。Directory Bucket： - -- 只在单个 AZ 内冗余。 -- 不支持 Versioning。 -- 删除后没有普通 S3 versioning 提供的恢复窗口。 - -当前 Cloud checker 还明确依赖： - -- S3Accessor::check_versioning() 要求 Versioning=Enabled。 -- get_life_cycle() 查找 NoncurrentVersionExpiration。 -- Checker 使用该保留天数做巡检间隔风险判断。 - -Directory Bucket 不支持这两个前提。简单地“跳过检查”会移除现有安全门禁；把 current-object Expiration 当成替代又可能删除仍然有效的 Doris 数据。因此 P0 必须在创建或修改 Storage Vault 时拒绝 Directory Bucket，而不是让 BE 写入成功后由 recycler/checker 失败。 - -只在 FE 校验不够：cloud/src/common/http_helper.cpp 暴露 add_s3_vault、alter_s3_vault 等直接 HTTP 路由，它们进入 cloud/src/meta-service/meta_service_resource.cpp 并可独立持久化 ObjectStoreInfoPB。P0 要在两层都执行同一 service-aware 门禁： - -1. FE 在生成 Storage Vault 请求前拒绝 official AWS service + 合法 --x-s3 bucket。 -2. Meta Service 在 ADD_S3_VAULT、ALTER_S3_VAULT 以及所有能直接创建或替换 S3 vault/object info 的 RPC/HTTP 路径中，必须在 txn->put 或修改 InstanceInfoPB 前再次拒绝。 -3. Meta Service 的判断必须同时使用 provider/service、endpoint authority/override mode 和完整 bucket；第三方 S3-compatible 服务中恰好以 --x-s3 结尾的 bucket 不能被误拒绝。 -4. 返回稳定的 INVALID_ARGUMENT/UnsupportedS3Surface 错误，不能先落库再依赖 recycler 或 checker 报错。 - -这是一项产品范围门禁，不改变现有 Cloud 信任边界或安全定级。 - -### 12.2 若产品决定支持 Cloud Vault - -必须单独通过以下设计门禁后才能进入 P1： - -1. 产品和运维明确接受单 AZ 与无 versioning 的数据耐久性、恢复和删除风险。 -2. cloud/src/recycler/s3_accessor.cpp 改用 S3ClientConfiguration、S3EndpointProvider、SDK endpoint rules、HTTPS 和 virtual-hosted。 -3. S3Conf::from_obj_store_info 当前把 OSS、S3、COS、OBS、BOS 都折叠成 S3Conf::S3；P1 必须保留原始 ObjectStoreInfoPB::Provider 或在折叠前派生 official AWS service kind，不能把第三方 vault 的 --x-s3 bucket 误判为 Directory Bucket。 -4. cloud/src/recycler/s3_obj_client.cpp 的 Put、DeleteObjects、List、Abort 使用 Directory Bucket checksum 和分页语义。 -5. 不把 session credentials 写入 ObjectStoreInfoPB 或 Meta Service。 -6. client/session cache 不能跨 instance、tenant、vault 或 credentials identity 共享。 -7. 重做 checker 的安全模型，不能仅 skip versioning/lifecycle： - - 明确 Directory Bucket 下不可恢复删除的告警和运维门禁。 - - lifecycle 只允许验证 AbortIncompleteMultipartUpload，不允许把通用 current-object Expiration 配在 Doris 数据根目录。 - - 为 checker 设计不依赖 noncurrent version retention 的独立巡检 SLA。 -8. recycler 的 mark-before-delete、两阶段删除、重试幂等和 packed-file 顺序保持不变。 -9. 完成 Cloud recycler、checker、snapshot、restore、storage vault 创建和滚动升级的端到端测试。 - -在这套安全模型评审完成前，Cloud 代码可以先复用 capability resolver 用于 fail fast，但不对外开放 Vault。 - -## 13. 安全设计 - -### 13.1 信任边界 - -按照 Doris threat model，外部 S3 配置由管理员信任，但 credentials、网络 endpoint 和 Cloud tenant 隔离仍属于必须保护的边界。 - -### 13.2 Credentials - -- 长期 AK/SK、AWS_TOKEN、role ARN 等继续走 Doris 现有安全配置和 credentials provider。 -- CreateSession 返回值只存在于 SDK 进程内存。 -- 不新增 session token 的 FE 属性、PB、Thrift、EditLog 或 Meta Service 字段。 -- 不记录 Authorization、x-amz-s3session-token、secret key、session token。 -- 修复 S3ClientConf::to_string() 对 token 的明文输出；external ID 也按敏感信息处理。 -- 日志可记录 bucket mode、region、zone-id、operation、HTTP status、AWS request id，不记录完整 credentials。 - -### 13.3 网络 - -- Directory Bucket 强制 HTTPS。 -- 不接受 path-style 或任意 endpoint override。 -- 保留 Doris 现有 endpoint 安全校验；官方 AWS host 仍需经过 allowlist/SSRF 规则。 -- 不跟随会把签名请求重定向到非 AWS host 的重定向。 - -### 13.4 Cloud tenant 隔离 - -若未来支持 Cloud： - -- cache key 包含 instance/vault/credentials identity。 -- 一个 tenant 的 session client 不得用于另一个 tenant，即使 bucket、region 和 endpoint 相同。 -- session 刷新失败只影响对应 client，不触发全局 credentials 降级。 - -## 14. 错误处理 - -建议新增或规范化以下用户可识别错误： - -| 错误 | 示例信息 | -| --- | --- | -| 非法 bucket 名 | AWS Directory Bucket requires a full bucket name ending with --x-s3 | -| 缺少 region | AWS Directory Bucket requires AWS_REGION | -| path-style | Path-style addressing is not supported for AWS Directory Bucket | -| HTTP | AWS Directory Bucket requires HTTPS | -| endpoint 冲突 | Configured endpoint region/zone does not match directory bucket | -| control endpoint | s3express-control endpoint cannot be used for object operations | -| session 权限 | Access denied while creating S3 Express session; grant s3express:CreateSession | -| 不支持的 surface | S3 Express One Zone is not supported by Hadoop S3A/Cloud Storage Vault in this release | -| checksum | CRC32C mismatch for UploadPart, bucket=..., key=..., part=... | -| multipart 空洞 | Directory Bucket multipart parts must be consecutive from 1 | -| list 资源不足 | Directory Bucket listing exceeded query memory limit while preserving ordered result | - -实现遵循 Doris 的“错误即失败”原则： - -- 配置不一致在创建 client 前失败。 -- checksum 不一致不能重试为无 checksum 请求。 -- session auth 失败不能回退为匿名或非 TLS。 -- Abort 失败不能被静默丢弃。 -- 只对 SDK 明确标记可重试的网络、限流和服务端错误使用现有 retry strategy。 - -## 15. 可观测性 - -### 15.1 日志 - -client 创建时记录： - -- bucket_type=general_purpose 或 directory -- service kind（official AWS 或 custom S3-compatible） -- region -- zone_id -- endpoint_mode=override 或 sdk_rules -- addressing=virtual 或 path -- checksum_policy - -请求失败记录： - -- operation -- bucket/key 的现有安全形式 -- HTTP status -- SDK exception name -- AWS request id -- 是否处于 Directory Bucket 模式 - -不在 INFO 级别记录每次成功请求或 session token。 - -### 15.2 Metrics - -建议在现有 S3 bvar 基础上增加低基数维度： - -- s3_client_create_total，按 bucket_type -- s3_request_total，按 bucket_type、operation、result -- s3_request_latency，按 bucket_type、operation -- s3_checksum_failure_total，按 operation -- s3_multipart_abort_total，按 result -- s3_directory_list_scanned_keys -- s3_directory_list_returned_keys -- s3_directory_list_pages -- s3_auth_failure_total，按 AccessDenied、ExpiredToken、其他 - -不要把 bucket 名、key 或 tenant id 作为 metric label。 - -AWS SDK 内部 CreateSession 不一定暴露独立回调；若不能可靠计数，不应猜测 session refresh 次数。可以通过 SDK debug 日志和长时集成测试验证刷新，通过外层 403/ExpiredToken 统计定位故障。 - -## 16. 测试方案 - -### 16.1 Capability resolver 单测 - -C++ 与 Java 使用同一组测试向量： - -- AWS + 合法 --x-s3。 -- bucket 中间包含 --x-s3 但后缀不匹配。 -- 自定义 S3-compatible endpoint + --x-s3。 -- AWS + custom endpoint。 -- zonal endpoint 的 region/zone 匹配与不匹配。 -- control endpoint。 -- 空 region。 -- HTTP。 -- path-style。 -- dualstack 官方 endpoint。 - -### 16.2 BE client 配置单测 - -通过可注入 builder 或 request hook 断言： - -- Directory Bucket：HTTPS、virtual、无 endpointOverride、disableS3ExpressAuth=false。 -- 普通 AWS S3：原 endpoint 和 addressing 行为不变。 -- S3-compatible：保留 endpointOverride，Express auth 禁用。 -- CA、aws_client_request_timeout_ms、resource timeout、connect timeout、max connections、retry strategy 被保留。 -- payload signing policy 与设计一致。 -- cache key 能隔离 need_override_endpoint、bucket type、scheme、checksum policy 和 credentials identity。 - -### 16.3 Checksum 单测 - -- CRC32C 已知向量：空字符串、123456789、小 buffer、跨 chunk buffer。 -- PutObject request： - - Directory 有 ChecksumAlgorithm=CRC32C 和正确 ChecksumCRC32C。 - - Directory 无 ContentMD5。 - - 普通 S3 仍有原 ContentMD5。 -- DeleteObjects request body 使用 CRC32C。 -- checksum mismatch 返回失败。 - -### 16.4 Multipart 单测 - -- Create 声明 CRC32C。 -- 多个并发 part 的 checksum 正确保存。 -- Complete 前排序。 -- Complete 携带每个 part 的 ETag 和 ChecksumCRC32C。 -- Hive BE-upload/FE-complete 路径把 checksum algorithm 和 part checksum 通过 TS3MPUPendingUpload 传到 FE。 -- Directory Bucket 下缺少可选 checksum 字段时拒绝 Complete 并 Abort。 -- part number 从 1 连续。 -- 空洞、重复、从 0 开始均失败。 -- UploadPart 失败时等待其他 future 后 Abort。 -- Complete 失败时 Abort。 -- Abort 的 NoSuchUpload 幂等成功。 -- 原始失败与 Abort 失败同时可见。 -- 普通 S3 和 Azure writer 不回归。 - -### 16.5 List 单测 - -构造无序、多页、超过 1000 个 key 的 mock 响应： - -- 不发送 StartAfter。 -- 服务 prefix 以 / 结尾。 -- 正确使用 NextContinuationToken。 -- 对 originalPrefix 和 glob 二次过滤。 -- 无序输入得到稳定有序输出。 -- startAfter 在本地应用。 -- 仅 maxFiles 时最多保留 N+1 个全局字典序最小对象，并生成正确 maxFile。 -- maxBytes 时先完整排序，包含触发阈值的对象，并生成正确 maxFile。 -- 未完成 multipart prefix 不被当成对象。 -- 空页但 IsTruncated=true 时继续翻页。 -- 递归删除每批删除后从空 token 重扫，不复用删除前的 opaque continuation token。 -- 前几页没有匹配 key 时继续扫描，不能误判删除完成。 -- 取消和内存不足返回错误。 - -### 16.6 FE 单测 - -- properties + URI 得到与 C++ 一致的 capability。 -- AWS_SDK_ROUTED 的 directory client 不 endpointOverride、不强制 signer、Express auth enabled；Directory request 在 path-style=true 时 fail fast。 -- general client 保留普通 AWS FIPS/dualstack/显式 endpoint 与 S3-compatible 既有行为。 -- AWS_EXPLICIT_ENDPOINT + Directory Bucket fail fast。 -- bucket property 为空时，从每次 remotePath 派生 capability；同一 S3ObjStorage 的 general/directory 两个 lazy client 不串配置。 -- custom endpoint 下以 --x-s3 结尾的 bucket 仍按 GENERAL_PURPOSE。 -- default chain、static session token、web identity、assume role provider 能传给 client。 -- Java PutObject、CreateMultipartUpload、UploadPart、CompleteMultipartUpload 的 CRC32C request/response 字段完整。 -- HMSTransaction 和 ObjFileSystem 正确消费 TS3MPUPendingUpload 的 checksum 字段；旧消息、缺字段和双失败路径有负例。 -- S3Resource ping 使用正确 Head/List 请求。 -- S3Resource ping 的业务失败与 Abort 失败同时可见，覆盖双失败测试。 -- Directory Bucket presign 返回 NotSupported。 -- 第 10.6 节调用面清单中的每一个入口都在“最早能取得完整 bucket 且尚未发出网络请求”的位置覆盖 Directory Bucket 负例，并保留普通 S3 正例；不能统一用一个抽象测试代替各入口门禁。 - -### 16.7 Cloud P0 门禁单测 - -- FE 创建和修改 Storage Vault 时拒绝 official AWS Directory Bucket。 -- 绕过 FE，直接调用 Meta Service ADD_S3_VAULT 和 ALTER_S3_VAULT 时，在任何持久化写入前返回 INVALID_ARGUMENT。 -- 覆盖其他能够直接创建或替换 S3 vault/object info 的 RPC/HTTP 路径，确认无法绕过同一门禁。 -- custom S3-compatible endpoint + --x-s3 后缀 bucket 不误拒绝。 -- 拒绝后 InstanceInfoPB、StorageVaultPB 和 recycler 可见状态均无变化。 - -### 16.8 AWS 真实集成测试 - -使用预创建的 Directory Bucket，凭证和 bucket 名通过 CI secret/environment 注入；测试默认关闭，只在专用 AWS job 运行。 - -必测： - -1. Put/Get/Head/Delete 小对象和空对象。 -2. 大对象 multipart，读取后逐字节校验。 -3. List 超过 1000 个 key、无序返回、glob、startAfter、maxFiles、maxBytes 和 maxFile。 -4. DeleteObjects。 -5. 主动 Abort 后确认 upload 不存在。 -6. 静态 IAM credentials。 -7. AssumeRole 或 web identity。 -8. 只读 policy 的读成功、写失败。 -9. 缺少 s3express:CreateSession 的明确 403。 -10. client 连续运行超过六分钟并跨越 session refresh。 -11. 并发首次请求，验证无 session refresh stampede 和请求失败。 -12. 错 region、HTTP、path-style、control endpoint 的负例。 -13. 同 AZ 和跨 AZ 只记录延迟差异，不设置易抖动的硬性能阈值。 - -普通 S3 和 MinIO 只能证明兼容性，不能替代 CreateSession 和 Directory Bucket 语义测试。 - -### 16.9 回归与工程检查 - -按改动阶段执行： - -- ./build.sh --be --fe -- run-be-ut.sh 运行相关 s3_util、s3_obj_storage_client、s3_file_writer 测试 -- run-fe-ut.sh 运行 S3URI、S3Properties、S3Util、S3ObjStorage 测试 -- run-regression-test.sh 运行 S3 Resource、TVF、外部文件读写相关 case -- build-support/clang-format.sh 格式化修改的 C++ 文件 -- build-support/check-format.sh 检查格式 -- BE build 生成 compile_commands.json 后，对修改文件运行 build-support/run-clang-tidy.sh - -若没有 Directory Bucket AWS job 通过，不得把 feature 标记为 GA。 - -## 17. 兼容性与滚动升级 - -### 17.1 既有工作负载 - -- 不改变普通 AWS S3 和 S3-compatible 的属性名称。 -- 不改变对象 key 或文件格式。 -- bucket type 不新增持久化字段；TS3MPUPendingUpload 只增加向后可解析的可选 checksum 字段。 -- 删除当前全局 s3_disable_content_md5，不让其改变既有 cached client。 -- 普通 S3 继续使用既有 Content-MD5 和 endpoint override 策略。 - -### 17.2 混合版本 - -旧 BE 无法可靠访问 Directory Bucket。滚动升级规则： - -1. 可以在混合版本期间继续使用普通 S3。 -2. 必须升级所有可能执行该资源 I/O 的 BE 和 FE 后，才能创建或启用 Directory Bucket Resource。 -3. TS3MPUPendingUpload 的新字段虽然是 optional，但旧 FE 会忽略 part checksum，旧 BE 也不会生成它；因此 optional 只保证协议解析，不保证 Directory Bucket 功能兼容。 -4. 若未来开放 Cloud Vault，所有 recycler/checker 实例必须先升级。 -5. 回滚到不支持版本前必须停止 Directory Bucket workload，并按第 20 节先把仍需访问的数据复制到普通 S3、切换到新资源；对象内容无需改格式，但旧版本不能读取留在 Directory Bucket 中的数据。 - -可选增强是增加 cluster capability bit，让 FE 在所有 BE 尚未声明 S3_DIRECTORY_BUCKET 时拒绝启用。若 Doris 当前没有适合的能力协商机制，首期通过发布文档和运维检查实现，不为此引入新的持久化协议。 - -## 18. 分阶段交付 - -### PR 1：SDK 基线与公共能力模型 - -内容： - -- 固定 AWS SDK C++ 1.11.400，并修正文档中的版本根因。 -- 保留 aws_logger vaLog 适配。 -- 增加 service-aware capability resolver 和跨语言测试向量。 -- 修复敏感 token 日志。 -- client config 从现有公共配置转换，补齐 cache key。 - -完成条件： - -- 普通 S3、MinIO 配置 UT 通过。 -- Directory Bucket client 配置 UT 通过。 -- 无业务 API 行为声明。 - -### PR 2：BE 读写完整链路 - -内容： - -- Directory Bucket endpoint/auth client。 -- PutObject CRC32C。 -- multipart CRC32C 全链路。 -- Hive BE-upload/FE-complete 的可选 Thrift checksum 传播。 -- AbortMultipartUpload。 -- DeleteObjects checksum。 -- exact-key 读写。 - -完成条件： - -- BE UT、真实 AWS 小对象和 multipart job 通过。 -- session refresh 超过六分钟通过。 -- 普通 S3、MinIO 回归通过。 - -### PR 3：List 与 FE - -内容： - -- S3CompatibleFileSystem 和 S3FileSystem 的 continuation token + 本地过滤排序算法。 -- S3ObjStorage、S3FileSystemProperties、S3Util、S3Resource builder 和 ping。 -- 完成第 10.6 节所有 Hadoop/S3A、Java v1、external catalog/scanner surface 清单，并逐入口 fail fast。 -- FE 与 Meta Service 在所有 Storage Vault 创建、修改入口执行 P0 fail fast。 -- Presigned URL 门禁。 - -完成条件： - -- 多页无序 List、glob、startAfter、maxFiles、maxBytes、maxFile 语义测试通过。 -- 调用面清单没有“待盘点”项，每个未支持入口都有普通 S3 正例和 Directory Bucket 负例。 -- 绕过 FE 直调 Meta Service 的 add/alter vault 测试证明拒绝发生在持久化前，且第三方 --x-s3 bucket 不误判。 -- FE UT、regression、真实 AWS 列举通过。 - -### PR 4：发布、文档与可观测性 - -内容： - -- 用户文档、IAM 示例、部署限制。 -- metrics 和错误信息。 -- mixed-version 与 rollback 文档。 -- AWS 专用 CI job 稳定运行。 - -完成条件： - -- P0 验收清单全部通过。 -- 发布说明明确 Cloud Vault、S3A、presign 等边界。 - -### 独立 P1：Cloud Storage Vault - -只有第 12 节的安全模型通过产品、Cloud 和运维评审后才启动，不能作为 P0 PR 的顺手补丁。 - -## 19. P0 验收标准 - -以下条件必须全部满足： - -1. 官方 AWS service + 合法 --x-s3 bucket 被稳定分类为 Directory Bucket；第三方兼容存储不误判。 -2. 线上请求使用 SDK 生成的 bucket-specific Zonal HTTPS virtual-host，不出现 path-style 或自定义 endpointOverride。 -3. Doris 未显式调用 CreateSession，但 AWS SDK 能自动创建、复用并刷新 session。 -4. client 连续运行跨越至少一次五分钟 session 过期仍能无中断读写。 -5. PutObject、UploadPart、CompleteMultipartUpload 和 DeleteObjects checksum 行为符合设计。 -6. multipart 任一失败会主动 Abort，不留下由正常失败路径产生的未完成 upload。 -7. ListObjectsV2 不发送 StartAfter，不依赖服务排序，多页结果符合 Doris 现有排序、startAfter、maxFiles、maxBytes 和 maxFile 契约。 -8. 普通 AWS S3、MinIO/S3-compatible、Azure writer 的现有测试不回归。 -9. FE 连通性检查使用相同 bucket 分类和 credentials provider。 -10. 第 10.6 节清单中的 Cloud Vault、S3A、Java v1、presign 等未支持路径都在最早确定点、首个网络请求或持久化之前 fail fast；FE 与 Meta Service 的直接 add/alter Vault 路径均有证据，第三方兼容存储不误判。 -11. 日志、metrics、PB/Thrift 中不存在 session secret 或明文 token。 -12. C++ format、clang-tidy、BE UT、FE UT、regression 和专用 AWS 集成测试通过。 -13. 用户文档明确单 AZ 故障域、同 AZ 部署建议、IAM、endpoint 和支持边界。 - -## 20. 回滚方案 - -- feature 不改变对象内容格式，但这不等于可以直接回滚：不支持该功能的旧 Doris binary 无法读取仍只存在于 Directory Bucket 的数据。 -- “停止流量/关闭新写入”和“恢复旧版本下的数据可用性”是两个不同层次。前者可以立即执行；后者必须完成数据复制和资源迁移后才能执行。 -- 运行时关闭方式是停止使用 Directory Bucket Resource，而不是动态切换 checksum 或禁用 session auth。 -- 为需要快速回滚的生产 workload，启用前应预先创建普通 S3 bucket、复制账号/权限、容量和校验清单；必要时在切换前安排持续复制或业务双写。P0 不在 Doris 内实现自动双写。 -- 若新版本出现 session、endpoint 或 list 问题，按以下顺序回滚： - 1. 停止所有新的 Directory Bucket 写入和会创建对象的作业，记录切流时间与最后成功提交点。 - 2. 等待正在进行的 multipart 完成；不能完成的 upload 使用仍受支持的新版本主动 Abort，并确认没有遗留 upload。 - 3. 在新版本 Doris/SDK 仍可访问 Directory Bucket 时，使用经过批准的外部复制工具或 Doris 导出路径，把仍需访问的对象复制到预建的普通 S3 bucket/root。保持对象 key，使用对象数量、字节数、清单和 checksum/逐对象读取校验复制完整性。 - 4. 新建指向普通 S3 bucket/root 的 S3 Resource 或 Storage Policy；不要试图原地修改旧 S3Resource 的 endpoint、region、root path 或 bucket，这些属性在当前实现中不可修改。 - 5. 使用 Doris 已支持的资源/策略/表迁移流程把 workload 切到新资源，执行代表性查询、写入和恢复检查；只有确认不再引用 Directory Bucket 后，才回滚 binary。 - 6. 保留 Directory Bucket 为只读观察窗口，达到既定保留期且清单复核完成后再删除数据。 -- P0 不支持 Cloud Storage Vault，因此上述切换只适用于本设计支持的外部 S3 Resource/FileSystem workload；不能把它解读为 Cloud 主存储的透明迁移方案。 -- 如果故障发生时没有可用的新版本读取路径，也没有预先复制的数据，只能停止流量，不能宣称完成了数据可用性回滚。 -- 不允许在同一个 Directory Bucket 上通过强制 endpointOverride 或 disableS3ExpressAuth 临时绕过故障，因为这可能产生错误签名、错误 host 或无效 checksum。 - -## 21. 待确认事项 - -下面事项不改变本文的 P0 架构决策，但必须在对应实现 PR 进入合并前关闭： - -1. SDK C++ 1.11.400 对 composite CRC32C 的具体 request/response model 字段，以编译和真实 AWS 测试为准。 -2. Java SDK v2 当前 pinned 版本在 Doris classloader 场景下的 S3 Express interceptor 加载行为。 -3. 第 10.6 节完整调用面清单、实际依赖版本和逐入口测试属于 PR 3 的阻断项，不能以“后续盘点”状态合入。 -4. presigned URL 是否有明确业务需求；若有，另写设计。 -5. Cloud Storage Vault 是否允许单 AZ、无 versioning 存储作为主数据层；默认答案为不允许。 -6. 是否增加 cluster capability bit 阻止混合版本启用 Directory Bucket。 - -## 22. 官方资料与源码依据 - -AWS： - -- [Differences for directory buckets](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-differences.html) -- [Directory bucket API operations](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-APIs.html) -- [Creating directory buckets in an Availability Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-create.html) -- [Networking for directory buckets in an Availability Zone](https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-az-networking.html) -- [S3 Express One Zone session authentication](https://docs.aws.amazon.com/sdkref/latest/guide/feature-s3-express.html) -- [Authenticating and authorizing requests](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-authenticating-authorizing.html) -- [CreateSession API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html) -- [PutObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) -- [DeleteObjects API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) -- [ListObjectsV2 API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) -- [Using multipart uploads with directory buckets](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-using-multipart-upload.html) -- [PutBucketLifecycleConfiguration API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) -- [Optimizing directory bucket performance](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-express-optimizing-performance.html) - -AWS SDK C++： - -- [AWS SDK C++ maintainer discussion: S3 Express customizations merged in 1.11.212](https://github.com/aws/aws-sdk-cpp/discussions/2623) -- [AWS SDK C++ 1.11.219 source tree](https://github.com/aws/aws-sdk-cpp/tree/1.11.219) -- [AWS SDK C++ S3ClientConfiguration](https://docs.aws.amazon.com/sdk-for-cpp/latest/api/aws-cpp-sdk-s3/html/struct_aws_1_1_s3_1_1_s3_client_configuration.html) -- [AWS SDK Java v2 S3BaseClientBuilder](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3BaseClientBuilder.html) - -Doris 当前实现入口： - -- be/src/util/s3_util.cpp -- be/src/util/s3_util.h -- be/src/io/fs/s3_obj_storage_client.cpp -- be/src/io/fs/s3_file_writer.cpp -- be/src/exec/sink/writer/vhive_partition_writer.cpp -- gensrc/thrift/DataSinks.thrift -- fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java -- fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java -- fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java -- fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java -- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java -- fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/FileSystem.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjStorage.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjectListOptions.java -- fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java -- fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java -- fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java -- fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulUtils.java -- fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java -- cloud/src/common/http_helper.cpp -- cloud/src/meta-service/meta_service_resource.cpp -- cloud/src/recycler/s3_accessor.cpp -- cloud/src/recycler/s3_obj_client.cpp -- cloud/src/recycler/checker.cpp diff --git a/s3_express_one_zone_implementation.md b/s3_express_one_zone_implementation.md deleted file mode 100644 index 21ecbc7bce034c..00000000000000 --- a/s3_express_one_zone_implementation.md +++ /dev/null @@ -1,534 +0,0 @@ -# Apache Doris S3 Express One Zone 实现改动梳理 - -## 1. 文档目的 - -本文梳理 `goal.md` 设计方案在提交 `8f732dce8a` 中的实际落地情况，重点回答： - -- Doris 为支持 S3 Express One Zone 实际修改了哪些模块。 -- Directory Bucket 请求与普通 S3 请求分别经过什么调用链。 -- checksum、multipart、ListObjectsV2 和失败清理如何处理。 -- 哪些 Doris 使用面已经接入原生实现，哪些使用面仍然主动拒绝。 -- 当前实现已经具备哪些代码能力，以及还缺少哪些编译、测试和真实 AWS 验证。 - -本文是实现状态说明，不替代 `goal.md` 中的完整背景、AWS 约束、发布标准和运维方案。 - -## 2. 实现状态 - -| 项目 | 内容 | -| --- | --- | -| 设计文档 | `goal.md` | -| 实现提交 | `8f732dce8a`，`[feature](s3) Support S3 Express One Zone directory buckets` | -| 当前阶段 | 代码已实现，尚未编译和执行测试 | -| 原生支持面 | BE S3 FileSystem、FE Java SDK v2 S3 FileSystem、S3 Resource 连通性检查、Hive multipart 提交链路 | -| 主动拒绝面 | Cloud Storage Vault、Hadoop/S3A 外部扫描、presigned URL、Java S3 CopyObject | -| 兼容目标 | 普通 AWS S3 与 MinIO、COS、OSS 等 S3-compatible 服务保持原有行为 | - -这里的“代码已实现”表示设计中的主要调用链已经进入代码，不表示已经满足 GA 验收。当前没有执行编译、单元测试、回归测试或真实 Directory Bucket 集成测试。 - -## 3. 总体设计 - -本次实现没有把 S3 Express 处理成一个全局开关，而是引入 request-level bucket capability： - -```text -bucket + endpoint + region + addressing mode - | - v - S3BucketCapabilities resolver - | - +---------+---------+ - | | - v v - GENERAL_PURPOSE DIRECTORY - | | - endpoint override SDK endpoint rules - Content-MD5 CRC32C - ordinary list continuation + local ordering - ordinary auth SDK-managed Express session auth -``` - -Directory Bucket 只有在以下条件同时成立时才会被识别： - -1. bucket 是合法的 Availability Zone Directory Bucket 完整名称，例如 - `analytics--use1-az4--x-s3`。 -2. endpoint 为空，或者属于官方 AWS S3 endpoint。 - -因此，在 MinIO 等自定义 endpoint 下，即使 bucket 名以 `--x-s3` 结尾，也仍按普通 S3-compatible bucket 处理。 - -## 4. 公共能力模型 - -### 4.1 C++ - -新增文件： - -- `common/cpp/s3_bucket_capabilities.h` - -核心类型： - -- `S3BucketType` - - `GENERAL_PURPOSE` - - `DIRECTORY` -- `S3EndpointMode` - - `EXPLICIT_OVERRIDE` - - `AWS_SDK_RULES` -- `S3ChecksumPolicy` - - `CONTENT_MD5` - - `CRC32C` -- `S3BucketCapabilities` - - 是否要求 HTTPS。 - - 是否要求 virtual-hosted addressing。 - - 是否支持 `StartAfter`。 - - 服务端结果是否保证字典序。 - - 是否支持 versioning 和 presign。 - -resolver 同时提供以下辅助校验： - -- Directory Bucket 名称和 zone-id 解析。 -- AWS S3、S3 Express Zonal 和 control endpoint 识别。 -- endpoint 中 region、zone-id 的解析与冲突检查。 -- FIPS、dualstack、accelerate、VPC endpoint 等显式 endpoint mode 识别。 - -### 4.2 Java - -新增文件： - -- `fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/S3BucketCapabilities.java` - -Java 侧镜像 C++ 的分类和配置校验规则，供 `fe-core`、`fe-filesystem-s3`、Hadoop FileSystem 和 connector 模块复用。 - -目前 C++ 和 Java 使用相同规则并分别有测试，但尚未按 `goal.md` 的建议抽取为一份由两端共同读取的测试向量文件。这是后续防止规则漂移的改进项。 - -## 5. BE 改动 - -### 5.1 S3ClientConf 与客户端缓存 - -修改文件： - -- `be/src/util/s3_util.h` -- `be/src/util/s3_util.cpp` - -主要改动： - -1. `S3ClientConf` 增加基于 bucket capability 的 client 行为派生。 -2. cache hash 改为有顺序的 hash combine，不再对所有字段直接 XOR。 -3. cache key 补充： - - bucket - - `need_override_endpoint` - - addressing mode - - credentials provider - - role ARN、external ID、token identity - - bucket type、endpoint mode、checksum policy - - 全局 HTTP scheme -4. `to_string()` 不再输出明文 token 和 external ID。 -5. 删除全局动态配置 `s3_disable_content_md5`，checksum 完全由 bucket capability 决定。 - -由于 bucket 进入 client cache key，不同 Directory Bucket 不会复用同一个 Doris cache entry；SDK 内部产生的 bucket-scoped session credentials 也不会进入 Doris 配置、日志或 RPC。 - -### 5.2 S3ClientFactory - -普通 S3 路径保持： - -- 原有 endpoint override。 -- 原有 virtual/path-style 行为。 -- `PayloadSigningPolicy::Never`。 -- Content-MD5。 - -Directory Bucket 路径改为： - -- 从 Doris 公共 `ClientConfiguration` 转换为 `S3ClientConfiguration`，保留 CA、timeout 和 retry 配置。 -- 不设置 `endpointOverride`。 -- 使用 `S3EndpointProvider` 和 SDK endpoint rules。 -- `disableS3ExpressAuth=false`，由 SDK 完成 CreateSession、session cache 和刷新。 -- `PayloadSigningPolicy::RequestDependent`。 -- 要求 region、HTTPS 和 virtual-hosted addressing。 -- 拒绝 path-style、HTTP、control endpoint、显式 FIPS/dualstack 等不支持配置，以及 region/zone 冲突。 - -自定义 S3-compatible endpoint 使用 `disableS3ExpressAuth=true`，避免 SDK 因特殊 bucket 名误启用 Express session auth。 - -### 5.3 特殊客户端创建入口 - -`be/src/exprs/function/ai/embed.h` 调整为先解析 URI 和 bucket，再创建 S3 client。Directory Bucket 的 AI media presigned URL 在创建 client 前返回 `NotSupported`。 - -`be/src/io/tools/file_cache_microbench.cpp` 补充 bucket 到 `S3ClientConf`，使 capability 和 cache key 能正确派生。 - -### 5.4 PutObject 与 UploadPart checksum - -修改文件： - -- `be/src/io/fs/s3_obj_storage_client.h` -- `be/src/io/fs/s3_obj_storage_client.cpp` - -`S3ObjStorageClient` 不再保存 `_disable_content_md5`，而是保存完整 `S3BucketCapabilities`。 - -Directory Bucket 写入行为： - -- `PutObject` 计算 Base64 CRC32C，设置 algorithm 和 checksum，不发送 Content-MD5。 -- `CreateMultipartUpload` 声明 `CRC32C`。 -- `UploadPart` 计算并发送 CRC32C。 -- `UploadPart` 校验响应中的 CRC32C；缺失或不一致均返回失败。 -- `CompleteMultipartUpload` 为每个 part 发送 ETag、part number 和 CRC32C。 -- Complete 前要求 Directory Bucket part number 为从 1 开始的连续序列。 - -普通 S3 继续发送 Content-MD5，不使用 Directory Bucket 分支。 - -### 5.5 Multipart 元数据与失败清理 - -修改文件： - -- `be/src/io/fs/obj_storage_client.h` -- `be/src/io/fs/s3_file_writer.cpp` - -新增数据： - -- `ObjectStorageUploadResponse.checksum_crc32c` -- `ObjectCompleteMultiPart.checksum_crc32c` - -`S3FileWriter` 会保存每个 part 的 CRC32C，在 Complete 前排序并检查 part 序列。 - -失败清理流程： - -```text -CreateMultipartUpload 成功 - | - v -并发 UploadPart / Complete / close - | - +---- 成功 ----> Complete - | - +---- 失败 - | - v - 等待所有异步 part 收敛 - | - v - AbortMultipartUpload -``` - -具体行为： - -- `ObjStorageClient` 增加 `abort_multipart_upload` 接口。 -- S3 实现调用 AWS `AbortMultipartUpload`。 -- 404/NoSuchUpload 作为幂等成功处理。 -- Abort 失败时保留原业务错误，并附加 cleanup 错误。 -- 析构函数不发起网络 Abort，只记录 warning 和 unfinished multipart metric。 - -### 5.6 DeleteObjects 与列举 - -Directory Bucket： - -- `DeleteObjects` 请求显式使用 CRC32C。 -- 批量删除响应中的所有 object error 都进入错误消息，不再只报告第一项。 -- 非 `/` 结尾的 list prefix 扩大到父目录 prefix。 -- 使用 continuation token 读取全部页面，再按原始 prefix 本地过滤。 -- BE 完整 list 结果在客户端按 key 排序。 -- 递归删除一旦删除了当前 batch，就丢弃旧 continuation token 并从头扫描，避免删除过程中继续使用失效的 opaque token。 - -### 5.7 BE 可观测性 - -新增指标： - -- CRC32C 校验失败数。 -- multipart Abort 次数和失败数。 -- writer 析构时仍存在 multipart upload 的次数。 -- Directory Bucket list page、扫描 key 和返回 key 数。 - -client 创建日志增加 bucket type、endpoint mode 和 checksum policy；credentials token 和 external ID 保持遮蔽。 - -## 6. Hive BE 上传、FE Complete 链路 - -Hive 写入不是在同一进程内完成 multipart：BE 上传 part，FE 在事务提交时 Complete。因此本次修改了 Thrift 协议。 - -修改文件： - -- `gensrc/thrift/DataSinks.thrift` -- `be/src/exec/sink/writer/vhive_partition_writer.cpp` -- `fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java` -- `fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/UploadPartResult.java` -- `fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/ObjFileSystem.java` - -新增可选字段： - -```thrift -enum TObjectStorageChecksumAlgorithm { - CRC32C = 0 -} - -5: optional TObjectStorageChecksumAlgorithm checksum_algorithm -6: optional map part_checksums -``` - -传播链路： - -```text -BE S3FileWriter.completed_parts - | - v -VHivePartitionWriter - | - v -TS3MPUPendingUpload - etags + algorithm + part_checksums - | - v -FE HMSTransaction - | - v -ObjFileSystem -> UploadPartResult - | - v -S3ObjStorage CompleteMultipartUpload -``` - -Directory Bucket Complete 缺少任一 part checksum 时会失败，并进入 Abort。普通 S3 和旧消息仍可不设置新字段。 - -这些字段是 optional，因此协议可以解析旧消息，但混合版本集群仍不能安全启用 Directory Bucket：旧 BE 不产生 checksum，旧 FE 也不会使用 checksum。 - -## 7. FE 原生 S3 FileSystem 改动 - -### 7.1 S3ObjStorage 客户端模型 - -修改文件： - -- `fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java` - -原来的单一 lazy client 改为： - -- `generalClient` -- `directoryClient` -- 两者共享同一个长期 `AwsCredentialsProvider` - -每个公开对象操作解析当前 URI 的 bucket，并通过 `clientFor(bucket)` 选择 client。bucket type 不会被第一次请求永久缓存。 - -general client： - -- 保留 endpoint override、path-style、signer 和兼容配置。 -- 自定义 endpoint 禁用 Express session auth。 - -directory client： - -- 不设置 endpoint override。 -- 强制 virtual-hosted addressing。 -- 启用 SDK Express session auth。 -- 复用原 credentials provider。 - -Java Directory Bucket 请求增加： - -- PutObject CRC32C。 -- CreateMultipartUpload CRC32C。 -- UploadPart CRC32C，并要求响应包含 `ChecksumCRC32C`。 -- CompleteMultipartUpload 传播每个 part 的 CRC32C。 -- DeleteObjects CRC32C。 -- Abort 404 幂等处理。 - -Java SDK 负责根据 request body 计算请求 checksum；当前 Java 代码要求 UploadPart 响应 checksum 存在并继续传给 Complete，但没有像 C++ 路径一样单独计算本地 CRC32C 后做值比较。 - -### 7.2 Directory Bucket glob - -修改文件： - -- `fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java` -- `fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java` - -Directory Bucket glob 算法： - -1. 原始非 glob prefix 截断到最后一个 `/`。 -2. 请求中不发送 `StartAfter`。 -3. 用 opaque continuation token 读取所有服务页面。 -4. 本地应用原始 prefix、glob 和 `key > startAfter` 过滤。 -5. 使用 Doris UTF-8 binary comparator 排序。 -6. 排序后应用 `maxFiles`、`maxBytes` 和 `maxFile` 契约。 - -递归删除在 Directory Bucket 上删除一个 batch 后重新从空 token 列举。 - -当前实现为保证语义正确，会收集全部匹配对象后排序；尚未实现仅有 `maxFiles` 时的有界堆优化，也没有新增 Directory Bucket 专用的内存限制和查询取消检查。 - -### 7.3 S3FileSystemProperties - -修改文件： - -- `fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java` - -增加 Directory Bucket region、endpoint、HTTPS 和 path-style 校验。将 Directory Bucket 属性转换为 Hadoop S3A 配置时直接拒绝，防止进入未经验证的 S3A client。 - -## 8. fe-core 改动 - -### 8.1 S3Util - -`fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java` 的多个 builder 收敛到统一内部实现： - -- 已知 bucket 时先解析 capability。 -- Directory Bucket 不设置 endpoint override，不强制 `AwsS3V4Signer`，启用 Express session auth。 -- 普通 S3 保留原 builder 行为。 -- 自定义 S3-compatible endpoint 禁用 Express session auth。 -- HeadBucket connectivity tester 传入完整 bucket。 - -### 8.2 S3Resource - -`fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java` 增加： - -- Directory Bucket endpoint 缺少 scheme 时默认补 HTTPS，而不是 HTTP。 -- 在网络请求前校验 region、endpoint、path-style 和 HTTPS。 -- ping list 使用父目录扫描，并翻页直到确认测试对象存在。 -- multipart Complete 失败时执行 Abort。 -- Abort 失败作为 suppressed exception 保留在原始错误中。 - -### 8.3 S3URI - -`S3URI.isS3DirectoryBucket` 改为复用统一的 Directory Bucket 名称校验，不再维护一套宽松的 suffix 分割规则。 - -## 9. 不支持使用面的 fail-fast - -### 9.1 Cloud Storage Vault - -修改文件： - -- `fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java` -- `cloud/src/meta-service/meta_service_resource.cpp` - -FE 在生成 create/alter vault 请求前拒绝 Directory Bucket。Meta Service 在直接 ADD/ALTER object info 或 Storage Vault 时再次校验，并在加密或持久化前返回 `INVALID_ARGUMENT`。 - -门禁同时要求： - -- provider 为 S3。 -- endpoint 是官方 AWS service。 -- bucket 是合法 Directory Bucket 名称。 - -因此第三方 S3-compatible 服务中的 `--x-s3` bucket 不会被误拒绝。 - -### 9.2 Hadoop/S3A 与外部扫描 - -以下入口在第一个 Hadoop/S3A 网络请求前拒绝 Directory Bucket： - -| 使用面 | 拒绝位置 | -| --- | --- | -| 通用 Hadoop FileSystem | `DFSFileSystem.getHadoopFs` | -| 新 Hive connector | `HiveScanPlanProvider` 创建 Hadoop FileSystem 前 | -| Hudi | `HudiScanNode` 创建 Hudi client 前 | -| LakeSoul | `LakeSoulScanNode` 初始化 scanner 前 | -| Paimon | 各 Paimon metastore `initializeCatalog` 前 | -| Avro S3 reader | `S3FileReader` 调用 `FileSystem.get` 前 | - -上述门禁都会结合 endpoint 判断官方 AWS 与自定义兼容服务，普通 S3 和自定义 endpoint 保持原路径。 - -### 9.3 Presign 和 Copy - -- Java `S3ObjStorage.getPresignedUrl` 对 Directory Bucket 返回不支持。 -- BE AI embed 入口明确返回 `NotSupported`。 -- BE 通用 `generate_presigned_url` 接口受限于返回类型，只能记录 warning 并返回空字符串。 -- Java `S3ObjStorage.copyObject` 对 Directory Bucket 返回不支持。 - -当前 BE `S3ObjStorageClient::copy_object` 尚未增加 Directory Bucket 显式门禁；Directory Bucket Access Point 也没有专门的资源标识识别。这两项不能写入 P0 支持声明，需要在合入前补齐或证明没有用户可达调用链。 - -## 10. 普通 S3 与兼容性 - -本次实现有意保留以下行为： - -- 普通 AWS S3 继续使用 Content-MD5。 -- 普通 S3 client 继续使用原 endpoint override 和 path-style 配置。 -- MinIO、COS、OSS 等自定义 endpoint 不根据 bucket suffix 启用 Express。 -- `UploadPartResult` 保留原两参数构造，OSS、COS、OBS、Azure 不需要 checksum 字段。 -- `ObjStorageClient.abort_multipart_upload` 默认是 no-op，避免要求所有已有 object storage 实现同时增加原生 Abort。 -- bucket capability 不增加 PB、EditLog 或持久化字段。 -- 只有 Hive multipart 的 Thrift 消息增加 optional checksum 字段。 - -## 11. 配置示例 - -```sql -CREATE RESOURCE "s3_express_hot" -PROPERTIES -( - "type" = "s3", - "provider" = "S3", - "AWS_ENDPOINT" = "https://s3.us-east-1.amazonaws.com", - "AWS_REGION" = "us-east-1", - "AWS_BUCKET" = "analytics-hot--use1-az4--x-s3", - "AWS_ROOT_PATH" = "hot-data/", - "AWS_ACCESS_KEY" = "", - "AWS_SECRET_KEY" = "", - "use_path_style" = "false" -); -``` - -关键约束： - -- bucket 必须使用完整 Directory Bucket 名称。 -- endpoint 推荐配置标准 regional HTTPS endpoint，不需要手工配置 bucket-specific Zonal endpoint。 -- region 必须与 bucket 所在 region 一致。 -- `use_path_style` 必须为 `false`。 -- IAM principal 必须拥有目标 bucket 的 `s3express:CreateSession` 权限。 -- Doris 只配置长期 credentials provider，不能配置或持久化 SDK 产生的 Express session token。 - -## 12. 已增加但未执行的测试 - -### 12.1 C++ - -- `be/test/util/s3_util_test.cpp` - - 官方 AWS Directory Bucket 分类。 - - 自定义 endpoint 负例。 - - endpoint mode、region 和 zone 解析。 -- `be/test/io/fs/s3_file_writer_test.cpp` - - Complete 失败后执行 Abort。 - -### 12.2 Java - -- `S3BucketCapabilitiesTest` - - 分类、region、zone、dualstack 和 custom endpoint。 -- `S3ObjStorageMockTest` - - Put、Create MPU、UploadPart、Complete 和 DeleteObjects CRC32C request model。 - - Directory Bucket list 不发送 StartAfter。 -- `S3FileSystemTest` - - 无序 list 的本地排序和 `maxFile`。 -- `S3FileSystemPropertiesTest` - - Directory Bucket 拒绝转换为 S3A。 -- `DFSFileSystemTest`、`HiveScanPlanProviderTest`、`S3UtilsTest` - - S3A/Avro fail-fast 和第三方 endpoint 兼容性。 - -这些测试文件已经进入提交，但遵照开发要求没有执行。 - -## 13. 尚未完成的验证和风险 - -以下项目在代码提交后仍然是明确的未完成项： - -1. 没有执行 BE、FE 或 Cloud 编译，接口签名和依赖闭包尚未由编译器验证。 -2. 没有运行任何单元测试、回归测试或静态分析。 -3. C++ clang-format 未完成；当前环境缺少 Homebrew/clang-format。 -4. 没有真实 AWS Directory Bucket 测试： - - CreateSession 权限和 403 错误。 - - 连续运行超过五分钟的 session refresh。 - - 并发首次请求和 refresh stampede。 - - Put、multipart、DeleteObjects 的实际 HTTP checksum header。 - - 无序、多页 ListObjectsV2。 -5. Java UploadPart 没有单独计算本地 CRC32C 并和响应值比较，目前依赖 SDK request checksum，并要求响应 checksum 存在。 -6. Directory glob 尚未增加专用内存上限、取消检查和 `maxFiles` 有界堆优化。 -7. 跨语言分类测试尚未使用同一个测试向量文件。 -8. BE CopyObject 和 Directory Bucket Access Point 缺少明确门禁。 -9. Cloud Vault 门禁缺少绕过 FE 直调 Meta Service 的自动化测试。 -10. 当前未增加 cluster capability bit，混合版本启用限制依靠发布文档和运维流程。 - -因此，在上述验证完成前，只能表述为“Directory Bucket 支持代码已实现”，不能表述为“功能已经通过验收”或“可以作为 GA 发布”。 - -## 14. 建议验证顺序 - -后续验证建议按以下顺序进行： - -1. 运行 C++ 和 Java 格式检查，先解决纯工程问题。 -2. 编译 Thrift、BE、FE 和 Cloud，确认跨模块接口完整。 -3. 运行 capability、S3 object client、file writer、filesystem 和 fail-fast 单测。 -4. 运行普通 AWS S3、MinIO/S3-compatible 回归，确认兼容路径没有变化。 -5. 使用真实 AWS Directory Bucket 验证小对象、multipart、Abort、DeleteObjects 和多页 list。 -6. 让 client 跨越至少一次五分钟 session 过期，并执行并发刷新验证。 -7. 验证 Cloud Meta Service 直接 add/alter vault 均在持久化前拒绝。 -8. 补齐 BE CopyObject、Access Point 等剩余边界后，再评估 P0 验收。 - -## 15. 回滚与混合版本 - -- 本次不改变对象 key 和对象内容格式。 -- 旧 Doris binary 不能可靠访问仍位于 Directory Bucket 中的数据，因此不能把二进制回滚等同于数据可用性回滚。 -- 回滚前必须停止 Directory Bucket workload，把仍需访问的数据复制到普通 S3，并切换 Resource。 -- 混合版本期间只能继续使用普通 S3；所有可能执行该资源 I/O 的 BE 和 FE 升级完成后，才能启用 Directory Bucket Resource。 -- Cloud Storage Vault 当前主动拒绝 Directory Bucket，不存在透明切换或回滚路径。 - -## 16. 结论 - -本次改动已经完成 S3 Express One Zone 的核心代码闭环：统一能力识别、SDK-managed session auth、CRC32C、multipart checksum 传播、失败 Abort、Directory Bucket list 语义、FE 原生 S3 client，以及未支持 surface 的 fail-fast。 - -当前最重要的下一步不是继续扩大支持面，而是完成编译、普通 S3 兼容回归和真实 Directory Bucket 集成验证，并补齐本文列出的 BE CopyObject、Access Point、资源限制和 Cloud 直调测试缺口。 diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 1b4ca87919f81b..af46e566b8a30f 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -356,10 +356,10 @@ BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css" BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a" # aws sdk -AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.400.tar.gz" -AWS_SDK_NAME="aws-sdk-cpp-1.11.400.tar.gz" -AWS_SDK_SOURCE="aws-sdk-cpp-1.11.400" -AWS_SDK_MD5SUM="329c46612df55ba6715d9d2ccd61dc03" +AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/refs/tags/1.11.219.tar.gz" +AWS_SDK_NAME="aws-sdk-cpp-1.11.219.tar.gz" +AWS_SDK_SOURCE="aws-sdk-cpp-1.11.219" +AWS_SDK_MD5SUM="80aa616efe1a3e7a9bf0dfbc44a97864" # tsan_header TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7" From 0a3e0262cc2e74718bded85dc90f1b85f01ddfa3 Mon Sep 17 00:00:00 2001 From: Refrain Date: Fri, 17 Jul 2026 17:14:01 +0800 Subject: [PATCH 08/13] [fix](s3) Align S3 Express support with known object paths ### What problem does this PR solve? Issue Number: None Related PR: #63409 Problem Summary: Align S3 Express support with the documented known-object contract. Only valid AWS Directory Buckets using a standard HTTPS regional endpoint, a matching region, and virtual-hosted-style access enable SDK endpoint rules and CreateSession. Exact S3 TVF and Broker Load paths use HeadObject and reject glob, range, query, empty-key, and directory-prefix inputs. Third-party S3-compatible endpoints retain conventional S3 authentication and checksum behavior even when a bucket name ends in --x-s3. BE known-object uploads omit Content-MD5 for Directory Buckets. Remove the prior generic Java operation and BE abort expansions. The existing SDK versions already provide the required endpoint and session APIs, so no SDK upgrade is included. ### Release note Support known-object access to Amazon S3 Express One Zone through native S3 paths. ### Check List (For Author) - Test: Not run (per user request) - Behavior changed: Yes. Valid AWS Directory Buckets use S3 Express session authentication for known-object access; unsupported listing and glob semantics are not enabled. - Does this need documentation: Yes. The user documentation is handled separately. --- be/src/io/fs/obj_storage_client.h | 5 - be/src/io/fs/s3_file_writer.cpp | 82 ++++------ be/src/io/fs/s3_obj_storage_client.cpp | 17 -- be/src/io/fs/s3_obj_storage_client.h | 1 - be/src/util/s3_util.cpp | 145 ++++++++++++++---- be/src/util/s3_util.h | 2 +- be/test/io/fs/s3_file_writer_test.cpp | 29 ---- be/test/io/s3_client_factory_test.cpp | 33 ++++ be/test/util/s3_util_test.cpp | 32 ++-- .../org/apache/doris/common/util/S3Util.java | 74 +++++---- .../load/loadv2/BrokerLoadPendingTask.java | 11 +- .../ExternalFileTableValuedFunction.java | 10 +- .../apache/doris/common/util/S3UtilTest.java | 26 +++- .../doris/filesystem/s3/S3ObjStorage.java | 84 +++++----- 14 files changed, 337 insertions(+), 214 deletions(-) diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h index 4a6b847ae30b10..fa239ca3282e2a 100644 --- a/be/src/io/fs/obj_storage_client.h +++ b/be/src/io/fs/obj_storage_client.h @@ -106,11 +106,6 @@ class ObjStorageClient { virtual ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) = 0; - // Abort an unfinished multipart upload. Backends without an explicit abort primitive may - // retain their existing no-op cleanup semantics. - virtual ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions&) { - return ObjectStorageResponse::OK(); - } // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. // If it exists, it will return the corresponding file size virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index c589759fd778bd..f8b836607a14a6 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -85,8 +85,7 @@ S3FileWriter::~S3FileWriter() { _wait_until_finish(fmt::format("wait s3 file {} upload to be finished", _obj_storage_path_opts.path.native())); } - // Do not issue network requests from the destructor. Close failures abort an active - // multipart upload in _close_impl(). + // We won't do S3 abort operation in BE, we let s3 service do it own. if (state() == State::OPENED && !_failed) { s3_bytes_written_total << _bytes_appended; } @@ -284,67 +283,40 @@ Status S3FileWriter::_submit_upload_buffer(const std::shared_ptr& bu Status S3FileWriter::_close_impl() { VLOG_DEBUG << "S3FileWriter::close, path: " << _obj_storage_path_opts.path.native(); - auto close_status = [&]() -> Status { - DBUG_EXECUTE_IF("S3FileWriter._close_impl.inject_error", { - if (_obj_storage_path_opts.key.ends_with(".dat")) { - return Status::IOError("S3FileWriter._close_impl.inject_error"); - } - }); - - if (_cur_part_num == 1 && - _pending_buf) { // data size is less than config::s3_write_buffer_size - RETURN_IF_ERROR(_set_upload_to_remote_less_than_buffer_size()); + DBUG_EXECUTE_IF("S3FileWriter._close_impl.inject_error", { + if (_obj_storage_path_opts.key.ends_with(".dat")) { + return Status::IOError("S3FileWriter._close_impl.inject_error"); } + }); - if (_bytes_appended == 0) { - DCHECK_EQ(_cur_part_num, 1); - // No data written, but need to create an empty file - RETURN_IF_ERROR(_build_upload_buffer()); - if (!_used_by_s3_committer) { - auto* pending_buf = dynamic_cast(_pending_buf.get()); - pending_buf->set_upload_to_remote( - [this](UploadFileBuffer& buf) { _put_object(buf); }); - } else { - RETURN_IF_ERROR(_create_multi_upload_request()); - } - } + if (_cur_part_num == 1 && _pending_buf) { // data size is less than config::s3_write_buffer_size + RETURN_IF_ERROR(_set_upload_to_remote_less_than_buffer_size()); + } - if (_pending_buf != nullptr) { // there is remaining data in buffer need to be uploaded - auto st = _submit_upload_buffer(_pending_buf); - _pending_buf = nullptr; - if (!st.ok()) { - _wait_until_finish("pending buffer submit failed"); - return st; - } + if (_bytes_appended == 0) { + DCHECK_EQ(_cur_part_num, 1); + // No data written, but need to create an empty file + RETURN_IF_ERROR(_build_upload_buffer()); + if (!_used_by_s3_committer) { + auto* pending_buf = dynamic_cast(_pending_buf.get()); + pending_buf->set_upload_to_remote([this](UploadFileBuffer& buf) { _put_object(buf); }); + } else { + RETURN_IF_ERROR(_create_multi_upload_request()); } + } - RETURN_IF_ERROR(_complete()); - SYNC_POINT_RETURN_WITH_VALUE("s3_file_writer::close", Status()); - return Status::OK(); - }(); - - if (!close_status.ok() && _obj_storage_path_opts.upload_id.has_value()) { - _wait_until_finish("abort after close failure"); - auto abort_status = _abort(); - if (!abort_status.ok()) { - close_status.append(fmt::format("; AbortMultipartUpload also failed: {}", - abort_status.to_string_no_stack())); + if (_pending_buf != nullptr) { // there is remaining data in buffer need to be uploaded + auto st = _submit_upload_buffer(_pending_buf); + _pending_buf = nullptr; + if (!st.ok()) { + _wait_until_finish("pending buffer submit failed"); + return st; } } - return close_status; -} -Status S3FileWriter::_abort() { - DCHECK(_obj_storage_path_opts.upload_id.has_value()); - const auto& client = _obj_client->get(); - if (client == nullptr) { - return Status::InternalError("invalid obj storage client while aborting MPU"); - } - auto resp = client->abort_multipart_upload(_obj_storage_path_opts); - if (resp.status.code != ErrorCode::OK) { - return {resp.status.code, std::move(resp.status.msg)}; - } - _obj_storage_path_opts.upload_id.reset(); + RETURN_IF_ERROR(_complete()); + SYNC_POINT_RETURN_WITH_VALUE("s3_file_writer::close", Status()); + return Status::OK(); } diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 0792046d320f9b..fe7440ab551bff 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -297,23 +297,6 @@ ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( return ObjectStorageResponse::OK(); } -ObjectStorageResponse S3ObjStorageClient::abort_multipart_upload( - const ObjectStoragePathOptions& opts) { - Aws::S3::Model::AbortMultipartUploadRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); - auto outcome = s3_put_rate_limit([&]() { return _client->AbortMultipartUpload(request); }); - if (outcome.IsSuccess() || - outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return ObjectStorageResponse::OK(); - } - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), - fmt::format("failed to AbortMultipartUpload: {}, upload_id={}", - opts.path.native(), *opts.upload_id))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; -} - ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { Aws::S3::Model::HeadObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 70088934afea25..770f1e15c4392f 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -45,7 +45,6 @@ class S3ObjStorageClient final : public ObjStorageClient { ObjectStorageResponse complete_multipart_upload( const ObjectStoragePathOptions& opts, const std::vector& completed_parts) override; - ObjectStorageResponse abort_multipart_upload(const ObjectStoragePathOptions& opts) override; ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, size_t offset, size_t bytes_read, diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index a484188ed89d73..919389796a7bd1 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -85,30 +85,74 @@ bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); namespace { -std::string s3_endpoint_host(std::string_view endpoint) { - const auto scheme = endpoint.find("://"); - if (scheme != std::string_view::npos) { - endpoint.remove_prefix(scheme + 3); +std::string ascii_lower(std::string_view value) { + std::string result(value); + std::transform(result.begin(), result.end(), result.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return result; +} + +struct AwsS3Endpoint { + bool aws_domain = false; + bool https = false; + bool standard_regional = false; + std::string region; +}; + +AwsS3Endpoint parse_aws_s3_endpoint(std::string_view endpoint) { + AwsS3Endpoint result; + const auto scheme_separator = endpoint.find("://"); + if (scheme_separator != std::string_view::npos) { + result.https = ascii_lower(endpoint.substr(0, scheme_separator)) == "https"; + endpoint.remove_prefix(scheme_separator + 3); + } + + const auto path_separator = endpoint.find_first_of("/?#"); + const auto authority = endpoint.substr(0, path_separator); + const auto path = path_separator == std::string_view::npos + ? std::string_view {} + : endpoint.substr(path_separator); + bool standard_port = true; + auto host = authority; + if (const auto port_separator = authority.rfind(':'); port_separator != std::string_view::npos) { + standard_port = authority.substr(port_separator + 1) == "443"; + host = authority.substr(0, port_separator); + } + + const auto lower_host = ascii_lower(host); + constexpr std::string_view aws_domain = ".amazonaws.com"; + constexpr std::string_view aws_china_domain = ".amazonaws.com.cn"; + constexpr std::string_view aws_dualstack_domain = ".api.aws"; + std::string_view domain; + if (lower_host.ends_with(aws_china_domain)) { + domain = aws_china_domain; + } else if (lower_host.ends_with(aws_domain)) { + domain = aws_domain; + } else if (lower_host.ends_with(aws_dualstack_domain)) { + result.aws_domain = true; + return result; + } else { + return result; } - endpoint = endpoint.substr(0, endpoint.find_first_of("/?#")); - if (const auto port = endpoint.find(':'); port != std::string_view::npos) { - endpoint = endpoint.substr(0, port); + result.aws_domain = true; + + constexpr std::string_view regional_prefix = "s3."; + if (!result.https || !standard_port || (!path.empty() && path != "/") || + !lower_host.starts_with(regional_prefix)) { + return result; } - std::string host(endpoint); - std::transform(host.begin(), host.end(), host.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return host; + const auto region = std::string_view(lower_host).substr( + regional_prefix.size(), lower_host.size() - regional_prefix.size() - domain.size()); + if (region.empty() || region.find('.') != std::string_view::npos) { + return result; + } + result.standard_regional = true; + result.region = region; + return result; } -bool is_official_aws_s3_endpoint(std::string_view endpoint) { - const auto host = s3_endpoint_host(endpoint); - const bool aws_dns = host.ends_with(".amazonaws.com") || - host.ends_with(".amazonaws.com.cn") || host.ends_with(".api.aws"); - return aws_dns && - (host.starts_with("s3.") || host.starts_with("s3-") || - host.starts_with("s3express-") || host.find(".s3.") != std::string::npos || - host.find(".s3-") != std::string::npos || - host.find(".s3express-") != std::string::npos); +bool has_s3_directory_bucket_suffix(std::string_view bucket) { + return bucket.ends_with("--x-s3"); } bool is_s3_directory_bucket_name(std::string_view bucket) { @@ -121,8 +165,22 @@ bool is_s3_directory_bucket_name(std::string_view bucket) { if (zone_separator == std::string_view::npos || zone_separator == 0) { return false; } + const auto base_name = bucket.substr(0, zone_separator); + if (!std::isalnum(static_cast(base_name.front())) || + !std::isalnum(static_cast(base_name.back())) || + !std::all_of(base_name.begin(), base_name.end(), [](unsigned char c) { + return std::islower(c) || std::isdigit(c) || c == '-'; + })) { + return false; + } const auto zone_id = bucket.substr(zone_separator + 2); - return zone_id.find("-az") != std::string_view::npos; + const auto az_separator = zone_id.rfind("-az"); + return az_separator != std::string_view::npos && az_separator > 0 && + az_separator + 3 < zone_id.size() && + std::all_of(zone_id.begin(), zone_id.begin() + az_separator, + [](unsigned char c) { return std::islower(c) || std::isdigit(c) || c == '-'; }) && + std::all_of(zone_id.begin() + az_separator + 3, zone_id.end(), + [](unsigned char c) { return std::isdigit(c); }); } doris::Status is_s3_conf_valid(const S3ClientConf& conf) { @@ -132,9 +190,31 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) { if (conf.region.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } - if (is_s3_express(conf.bucket, conf.endpoint) && !conf.use_virtual_addressing) { - return Status::InvalidArgument( - "Path-style addressing is not supported for AWS Directory Bucket"); + const auto endpoint = parse_aws_s3_endpoint(conf.endpoint); + if (endpoint.aws_domain && has_s3_directory_bucket_suffix(conf.bucket)) { + if (!is_s3_directory_bucket_name(conf.bucket)) { + return Status::InvalidArgument( + "Invalid AWS Directory Bucket name {}; expected " + "----x-s3", + conf.bucket); + } + if (!endpoint.https) { + return Status::InvalidArgument("AWS Directory Bucket requires HTTPS"); + } + if (!endpoint.standard_regional) { + return Status::InvalidArgument( + "AWS Directory Bucket requires a standard regional S3 endpoint; do not use " + "s3express-control or a zonal endpoint"); + } + if (endpoint.region != ascii_lower(conf.region)) { + return Status::InvalidArgument( + "AWS Directory Bucket endpoint region {} does not match configured region {}", + endpoint.region, conf.region); + } + if (!conf.use_virtual_addressing) { + return Status::InvalidArgument( + "Path-style addressing is not supported for AWS Directory Bucket"); + } } if (conf.role_arn.empty()) { @@ -535,7 +615,8 @@ std::shared_ptr S3ClientFactory::get_aws_cred std::shared_ptr S3ClientFactory::_create_s3_client( const S3ClientConf& s3_conf) { - const bool s3_express = is_s3_express(s3_conf.bucket, s3_conf.endpoint); + const bool s3_express = + is_s3_express(s3_conf.bucket, s3_conf.endpoint, s3_conf.region); TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", std::make_shared(std::make_shared())); @@ -586,6 +667,14 @@ std::shared_ptr S3ClientFactory::_create_s3_client( new_client = std::make_shared( get_aws_credentials_provider(s3_conf), Aws::MakeShared("S3Client"), express_config); + } else if (has_s3_directory_bucket_suffix(s3_conf.bucket)) { + Aws::S3::S3ClientConfiguration compatible_config( + aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + s3_conf.use_virtual_addressing); + compatible_config.disableS3ExpressAuth = true; + new_client = std::make_shared( + get_aws_credentials_provider(s3_conf), + Aws::MakeShared("S3Client"), compatible_config); } else { new_client = std::make_shared( get_aws_credentials_provider(s3_conf), std::move(aws_config), @@ -856,8 +945,10 @@ std::string hide_access_key(const std::string& ak) { return key; } -bool is_s3_express(std::string_view bucket, std::string_view endpoint) { - return is_s3_directory_bucket_name(bucket) && is_official_aws_s3_endpoint(endpoint); +bool is_s3_express(std::string_view bucket, std::string_view endpoint, std::string_view region) { + const auto parsed_endpoint = parse_aws_s3_endpoint(endpoint); + return is_s3_directory_bucket_name(bucket) && parsed_endpoint.standard_regional && + parsed_endpoint.region == ascii_lower(region); } } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index e73dc3a9dbcae4..d459a70ebaa7c2 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -64,7 +64,7 @@ extern bvar::LatencyRecorder s3_copy_object_latency; }; // namespace s3_bvar std::string hide_access_key(const std::string& ak); -bool is_s3_express(std::string_view bucket, std::string_view endpoint); +bool is_s3_express(std::string_view bucket, std::string_view endpoint, std::string_view region); int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); // Rebuild the S3 GET/PUT rate limiters if the related configs have changed. // Safe to call periodically; it is a no-op when nothing changed. diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 0d54dbdda86b01..13f9d12c501197 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -1140,9 +1140,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { const std::vector& completed_parts) override { std::lock_guard lock(_mutex); complete_multipart_count++; - if (fail_complete_multipart) { - return {.status = {.code = 1, .msg = "injected complete multipart failure"}}; - } complete_multipart_params.push_back({opts, completed_parts}); last_opts = opts; last_completed_parts = completed_parts; @@ -1156,14 +1153,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageResponse abort_multipart_upload( - const ObjectStoragePathOptions& opts) override { - std::lock_guard lock(_mutex); - abort_multipart_count++; - last_opts = opts; - return default_response; - } - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { std::lock_guard lock(_mutex); return {.resp = ObjectStorageResponse::OK(), @@ -1238,8 +1227,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { int put_object_count = 0; int upload_part_count = 0; int complete_multipart_count = 0; - int abort_multipart_count = 0; - bool fail_complete_multipart = false; // Structures to store input parameters for each call struct UploadPartParams { @@ -1278,8 +1265,6 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { put_object_count = 0; upload_part_count = 0; complete_multipart_count = 0; - abort_multipart_count = 0; - fail_complete_multipart = false; create_multipart_params.clear(); put_object_params.clear(); @@ -1321,20 +1306,6 @@ create_s3_client(const std::string& path) { return {mock_client, s3_file_writer}; } -TEST_F(S3FileWriterTest, abortMultipartUploadWhenCompleteFails) { - auto [client, writer] = create_s3_client("abort_on_complete_failure"); - std::string content(config::s3_write_buffer_size + 1, 'x'); - ASSERT_TRUE(writer->append(Slice(content)).ok()); - client->fail_complete_multipart = true; - - auto status = writer->close(); - - EXPECT_FALSE(status.ok()); - EXPECT_EQ(1, client->complete_multipart_count); - EXPECT_EQ(1, client->abort_multipart_count); - EXPECT_TRUE(writer->upload_id().empty()); -} - /** * Generate test data for S3FileWriter boundary tests. * Returns a vector of sizes that we'll use to generate data on demand. diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 792d815cbca5f8..2b37161289366b 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -234,6 +234,39 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfCredentialValidation) { } } +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ExpressConf) { + S3URI s3_uri("s3://analytics--usw2-az1--x-s3/path/to/data.parquet"); + ASSERT_TRUE(s3_uri.parse().ok()); + + std::map properties { + {"AWS_ENDPOINT", "https://s3.us-west-2.amazonaws.com"}, + {"AWS_REGION", "us-west-2"}, + }; + S3Conf s3_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + + properties["AWS_ENDPOINT"] = "http://s3.us-west-2.amazonaws.com"; + ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + + properties["AWS_ENDPOINT"] = "https://s3express-control.us-west-2.amazonaws.com"; + ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + + properties["AWS_ENDPOINT"] = + "https://analytics--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"; + ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + + properties["AWS_ENDPOINT"] = "https://s3.us-west-2.amazonaws.com"; + properties["AWS_REGION"] = "us-east-1"; + ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + + properties["AWS_REGION"] = "us-west-2"; + properties["use_path_style"] = "true"; + ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + + properties["AWS_ENDPOINT"] = "https://minio.example.com"; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) { S3ClientFactory& factory = S3ClientFactory::instance(); config::aws_credentials_provider_version = "v2"; diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index f18111df2e55cf..45e9030fec52fa 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -79,21 +79,33 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { TEST_F(S3UTILTest, is_s3_express_context) { EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", - "https://s3.us-east-1.amazonaws.com")); - EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", - "bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1." - "amazonaws.com")); + "https://s3.us-east-1.amazonaws.com", "us-east-1")); EXPECT_TRUE(is_s3_express("bucket--cnn1-az1--x-s3", - "https://s3.cn-north-1.amazonaws.com.cn")); + "https://s3.cn-north-1.amazonaws.com.cn", "cn-north-1")); // S3-compatible services must retain their existing endpoint and checksum behavior, // even if a bucket happens to use the Directory Bucket suffix. - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com", + "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", + "https://s3.us-east-1.amazonaws.com.example.com", "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--x-s3", "https://s3.us-east-1.amazonaws.com", + "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--zone--x-s3", "https://s3.us-east-1.amazonaws.com", + "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path", "us-east-1")); + + // Users configure the standard HTTPS regional endpoint. The SDK derives the zonal endpoint. + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", + "http://s3.us-east-1.amazonaws.com", "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", + "https://s3.us-east-1.amazonaws.com", "us-west-2")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", + "https://s3express-control.us-east-1.amazonaws.com", "us-east-1")); EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", - "https://s3.us-east-1.amazonaws.com.example.com")); - EXPECT_FALSE(is_s3_express("bucket--x-s3", "https://s3.us-east-1.amazonaws.com")); - EXPECT_FALSE(is_s3_express("bucket--zone--x-s3", "https://s3.us-east-1.amazonaws.com")); - EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path")); + "https://bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1." + "amazonaws.com", + "us-east-1")); } // Verifies that check_s3_rate_limiter_config_changed() rebuilds the global GET rate diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index 15ca49a51297b0..568a37b0f62a85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -64,6 +64,12 @@ public class S3Util { private static final Logger LOG = LogManager.getLogger(Util.class); + private static final String DIRECTORY_BUCKET_SUFFIX = "--x-s3"; + private static final Pattern DIRECTORY_BUCKET_PATTERN = + Pattern.compile("^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9-]+-az[0-9]+--x-s3$"); + private static final Pattern AWS_S3_REGIONAL_ENDPOINT_PATTERN = Pattern.compile( + "^https://s3\\.([a-z0-9-]+)\\.amazonaws\\.com(?:\\.cn)?(?::443)?/?$", + Pattern.CASE_INSENSITIVE); private static AwsCredentialsProvider getAwsCredencialsProvider(CloudCredential credential) { AwsCredentials awsCredential; @@ -303,45 +309,61 @@ public static String getLongestPrefix(String globPattern) { return globPattern.substring(0, earliestSpecialCharIndex); } - public static boolean isExactS3ExpressObject(String path, String endpoint) throws UserException { + public static boolean isExactS3ExpressObject( + String path, String endpoint, String region, boolean usePathStyle) throws UserException { if (!(path.regionMatches(true, 0, "s3://", 0, "s3://".length()) || path.regionMatches(true, 0, "s3a://", 0, "s3a://".length()) - || path.regionMatches(true, 0, "s3n://", 0, "s3n://".length())) - || !getLongestPrefix(path).equals(path)) { + || path.regionMatches(true, 0, "s3n://", 0, "s3n://".length()))) { return false; } - return isS3Express(S3URI.create(path).getBucket(), endpoint); - } - - private static boolean isS3Express(String bucketName, String endpoint) { - if (!isDirectoryBucketName(bucketName) || Strings.isNullOrEmpty(endpoint)) { + S3URI uri = S3URI.create(path); + String bucketName = uri.getBucket(); + if (!bucketName.endsWith(DIRECTORY_BUCKET_SUFFIX) || Strings.isNullOrEmpty(endpoint)) { return false; } - String endpointUri = endpoint.contains("://") ? endpoint : "https://" + endpoint; - String host = URI.create(endpointUri).getHost(); - if (host == null) { + Matcher awsEndpoint = AWS_S3_REGIONAL_ENDPOINT_PATTERN.matcher(endpoint); + if (!awsEndpoint.matches()) { + if (isAwsEndpoint(endpoint)) { + throw new UserException("AWS Directory Bucket requires a standard HTTPS regional S3 " + + "endpoint; do not use s3express-control or a zonal endpoint"); + } return false; } - host = host.toLowerCase(Locale.ROOT); - boolean awsDns = host.endsWith(".amazonaws.com") - || host.endsWith(".amazonaws.com.cn") - || host.endsWith(".api.aws"); - return awsDns && (host.startsWith("s3.") - || host.startsWith("s3-") - || host.startsWith("s3express-") - || host.contains(".s3.") - || host.contains(".s3-") - || host.contains(".s3express-")); + if (!isDirectoryBucketName(bucketName)) { + throw new UserException("Invalid AWS Directory Bucket name " + bucketName + + "; expected ----x-s3"); + } + if (!awsEndpoint.group(1).equalsIgnoreCase(region)) { + throw new UserException("AWS Directory Bucket endpoint region " + awsEndpoint.group(1) + + " does not match configured region " + region); + } + if (usePathStyle) { + throw new UserException("Path-style addressing is not supported for AWS Directory Bucket"); + } + if (Strings.isNullOrEmpty(uri.getKey()) || !getLongestPrefix(path).equals(path) + || uri.getKey().endsWith("/") + || uri.getQueryParams().isPresent()) { + throw new UserException("AWS Directory Bucket TVF and Load paths must identify one complete " + + "object key and cannot contain glob expressions or a directory prefix"); + } + return true; } private static boolean isDirectoryBucketName(String bucketName) { - String suffix = "--x-s3"; - if (bucketName == null || !bucketName.endsWith(suffix)) { + return bucketName != null && DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches(); + } + + private static boolean isAwsEndpoint(String endpoint) { + int schemeSeparator = endpoint.indexOf("://"); + URI uri = URI.create(schemeSeparator > 0 ? endpoint : "https://" + endpoint); + String host = uri.getHost(); + if (host == null) { return false; } - String prefix = bucketName.substring(0, bucketName.length() - suffix.length()); - int zoneSeparator = prefix.lastIndexOf("--"); - return zoneSeparator > 0 && prefix.substring(zoneSeparator + 2).contains("-az"); + host = host.toLowerCase(Locale.ROOT); + return host.endsWith(".amazonaws.com") + || host.endsWith(".amazonaws.com.cn") + || host.endsWith(".api.aws"); } // Apply some rules to extend the globs parsing behavior diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java index ebfd7f7119b3d6..98a176e25041ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java @@ -111,8 +111,8 @@ protected void getAllFileStatus() throws UserException { // prefix-siblings (e.g. "file.csv.bz2" when listing "file.csv"). List entries; if (storageProperties instanceof ObjectStorageProperties - && S3Util.isExactS3ExpressObject(path, - ((ObjectStorageProperties) storageProperties).getEndpoint())) { + && isExactS3ExpressObject( + path, (ObjectStorageProperties) storageProperties)) { Location location = Location.of(path); entries = List.of(new FileEntry( location, fs.newInputFile(location).length(), false, 0L, List.of())); @@ -186,4 +186,11 @@ protected void getAllFileStatus() throws UserException { ((BrokerLoadJob) callback).setLoadFileInfo(totalFileNum, totalFileSize); } + + private static boolean isExactS3ExpressObject(String path, ObjectStorageProperties properties) + throws UserException { + return S3Util.isExactS3ExpressObject( + path, properties.getEndpoint(), properties.getRegion(), + Boolean.parseBoolean(properties.getUsePathStyle())); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 695a4b8fc5c903..5d673365cc96b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -167,8 +167,7 @@ protected void parseFile() throws AnalysisException { try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(brokerDesc)) { List entries; if (sp instanceof ObjectStorageProperties - && S3Util.isExactS3ExpressObject( - path, ((ObjectStorageProperties) sp).getEndpoint())) { + && isExactS3ExpressObject(path, (ObjectStorageProperties) sp)) { Location location = Location.of(path); entries = List.of(new FileEntry( location, fs.newInputFile(location).length(), false, 0L, List.of())); @@ -200,6 +199,13 @@ protected void parseFile() throws AnalysisException { } } + private static boolean isExactS3ExpressObject(String path, ObjectStorageProperties properties) + throws UserException { + return S3Util.isExactS3ExpressObject( + path, properties.getEndpoint(), properties.getRegion(), + Boolean.parseBoolean(properties.getUsePathStyle())); + } + // The keys in properties map need to be lowercase. protected Map parseCommonProperties(Map properties) throws AnalysisException { diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java index 4f0dd06df7e80f..41b374fea0bb9e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java @@ -17,6 +17,8 @@ package org.apache.doris.common.util; +import org.apache.doris.common.UserException; + import org.junit.Assert; import org.junit.Test; @@ -30,12 +32,28 @@ public void testIsExactS3ExpressObject() throws Exception { String object = "s3://analytics--use1-az4--x-s3/data/file.parquet"; String awsEndpoint = "https://s3.us-east-1.amazonaws.com"; - Assert.assertTrue(S3Util.isExactS3ExpressObject(object, awsEndpoint)); + Assert.assertTrue(S3Util.isExactS3ExpressObject( + object, awsEndpoint, "us-east-1", false)); Assert.assertFalse(S3Util.isExactS3ExpressObject( - "s3://analytics--use1-az4--x-s3/data/*.parquet", awsEndpoint)); - Assert.assertFalse(S3Util.isExactS3ExpressObject(object, "https://minio.example.com")); + object, "https://minio.example.com", "us-east-1", false)); Assert.assertFalse(S3Util.isExactS3ExpressObject( - "s3://analytics/data/file.parquet", awsEndpoint)); + "s3://analytics/data/file.parquet", awsEndpoint, "us-east-1", false)); + + Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( + "s3://analytics--use1-az4--x-s3/data/*.parquet", + awsEndpoint, "us-east-1", false)); + Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( + "s3://analytics--use1-az4--x-s3/data/", + awsEndpoint, "us-east-1", false)); + Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( + "s3://analytics--use1-az4--x-s3", + awsEndpoint, "us-east-1", false)); + Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( + object, "https://s3express-control.us-east-1.amazonaws.com", "us-east-1", false)); + Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( + object, awsEndpoint, "us-west-2", false)); + Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( + object, awsEndpoint, "us-east-1", true)); } @Test diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index ce2616432a0ebb..402a65e4c42e98 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -80,6 +80,8 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -90,6 +92,12 @@ public class S3ObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(S3ObjStorage.class); + private static final String DIRECTORY_BUCKET_SUFFIX = "--x-s3"; + private static final Pattern DIRECTORY_BUCKET_PATTERN = + Pattern.compile("^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9-]+-az[0-9]+--x-s3$"); + private static final Pattern AWS_S3_REGIONAL_ENDPOINT_PATTERN = Pattern.compile( + "^https://s3\\.([a-z0-9-]+)\\.amazonaws\\.com(?:\\.cn)?(?::443)?/?$", + Pattern.CASE_INSENSITIVE); /** Validity period for pre-signed URLs and STS tokens (seconds). */ private static final int SESSION_EXPIRE_SECONDS = 3600; @@ -158,7 +166,7 @@ private S3Client buildClient(String endpointStr, String region, .httpClient(UrlConnectionHttpClient.builder() .socketTimeout(Duration.ofSeconds(30)) .connectionTimeout(Duration.ofSeconds(30)) - .build()) + .build()) .credentialsProvider(clientCredentialsProvider) .region(Region.of(region)) .disableS3ExpressSessionAuth(!express) @@ -203,10 +211,30 @@ private AwsCredentialsProvider getCredentialsProvider() { return credentialsProvider; } - private S3Client clientFor(String requestBucket) throws IOException { - if (!isS3Express(requestBucket, s3Properties.getEndpoint())) { + private S3Client clientForHead(String requestBucket) throws IOException { + if (!requestBucket.endsWith(DIRECTORY_BUCKET_SUFFIX) + || StringUtils.isBlank(s3Properties.getEndpoint())) { return getClient(); } + Matcher endpoint = AWS_S3_REGIONAL_ENDPOINT_PATTERN.matcher(s3Properties.getEndpoint()); + if (!endpoint.matches()) { + if (isAwsEndpoint(s3Properties.getEndpoint())) { + throw new IOException("AWS Directory Bucket requires a standard HTTPS regional S3 " + + "endpoint; do not use s3express-control or a zonal endpoint"); + } + return getClient(); + } + if (!isDirectoryBucketName(requestBucket)) { + throw new IOException("Invalid AWS Directory Bucket name " + requestBucket + + "; expected ----x-s3"); + } + if (!endpoint.group(1).equalsIgnoreCase(s3Properties.getRegion())) { + throw new IOException("AWS Directory Bucket endpoint region " + endpoint.group(1) + + " does not match configured region " + s3Properties.getRegion()); + } + if (usePathStyle) { + throw new IOException("Path-style addressing is not supported for AWS Directory Bucket"); + } if (closed.get()) { throw new IOException("S3ObjStorage is already closed"); } @@ -323,7 +351,7 @@ public RemoteObjects listObjectsNonRecursive(String remotePath, String continuat public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - HeadObjectResponse response = clientFor(uri.bucket()).headObject( + HeadObjectResponse response = clientForHead(uri.bucket()).headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return new org.apache.doris.filesystem.spi.RemoteObject( uri.key(), uri.key(), response.eTag(), response.contentLength(), @@ -345,7 +373,7 @@ public void putObject(String remotePath, org.apache.doris.filesystem.spi.Request throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try (InputStream content = requestBody.content()) { - clientFor(uri.bucket()).putObject( + getClient().putObject( PutObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build(), software.amazon.awssdk.core.sync.RequestBody.fromInputStream( content, requestBody.contentLength())); @@ -391,7 +419,7 @@ public void copyObject(String srcPath, String dstPath) throws IOException { public String initiateMultipartUpload(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - CreateMultipartUploadResponse response = clientFor(uri.bucket()).createMultipartUpload( + CreateMultipartUploadResponse response = getClient().createMultipartUpload( CreateMultipartUploadRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return response.uploadId(); } catch (SdkException e) { @@ -405,7 +433,7 @@ public UploadPartResult uploadPart(String remotePath, String uploadId, int partN org.apache.doris.filesystem.spi.RequestBody body) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try (InputStream content = body.content()) { - UploadPartResponse response = clientFor(uri.bucket()).uploadPart( + UploadPartResponse response = getClient().uploadPart( UploadPartRequest.builder() .bucket(uri.bucket()).key(uri.key()) .uploadId(uploadId).partNumber(partNum) @@ -428,7 +456,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, .map(p -> CompletedPart.builder().partNumber(p.partNumber()).eTag(p.etag()).build()) .collect(Collectors.toList()); try { - clientFor(uri.bucket()).completeMultipartUpload(CompleteMultipartUploadRequest.builder() + getClient().completeMultipartUpload(CompleteMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId) .multipartUpload(CompletedMultipartUpload.builder().parts(completedParts).build()) .build()); @@ -442,7 +470,7 @@ public void completeMultipartUpload(String remotePath, String uploadId, public void abortMultipartUpload(String remotePath, String uploadId) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - clientFor(uri.bucket()).abortMultipartUpload(AbortMultipartUploadRequest.builder() + getClient().abortMultipartUpload(AbortMultipartUploadRequest.builder() .bucket(uri.bucket()).key(uri.key()).uploadId(uploadId).build()); } catch (S3Exception e) { // Re-throw so callers know the abort failed; orphaned parts may still exist @@ -462,7 +490,7 @@ public void abortMultipartUpload(String remotePath, String uploadId) throws IOEx InputStream openInputStream(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - return clientFor(uri.bucket()).getObject( + return getClient().getObject( GetObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); } catch (NoSuchKeyException e) { throw new FileNotFoundException("Object not found: " + remotePath); @@ -483,7 +511,7 @@ public InputStream openInputStreamAt(String remotePath, long fromByte) throws IO if (fromByte > 0) { req.range("bytes=" + fromByte + "-"); } - return clientFor(uri.bucket()).getObject(req.build()); + return getClient().getObject(req.build()); } catch (NoSuchKeyException e) { throw new FileNotFoundException("Object not found: " + remotePath); } catch (SdkException e) { @@ -498,7 +526,7 @@ public InputStream openInputStreamAt(String remotePath, long fromByte) throws IO public long headObjectLastModified(String remotePath) throws IOException { S3Uri uri = S3Uri.parse(remotePath, usePathStyle); try { - HeadObjectResponse resp = clientFor(uri.bucket()).headObject( + HeadObjectResponse resp = getClient().headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return resp.lastModified() != null ? resp.lastModified().toEpochMilli() : 0L; } catch (NoSuchKeyException e) { @@ -687,35 +715,21 @@ private static String getRelativePathSafe(String prefix, String key) { return key.substring(normalized.length()); } - private static boolean isS3Express(String bucketName, String endpoint) { - if (!isDirectoryBucketName(bucketName) || StringUtils.isBlank(endpoint)) { - return false; - } - String endpointUri = endpoint.contains("://") ? endpoint : "https://" + endpoint; - String host = URI.create(endpointUri).getHost(); + private static boolean isDirectoryBucketName(String bucketName) { + return bucketName != null && DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches(); + } + + private static boolean isAwsEndpoint(String endpoint) { + int schemeSeparator = endpoint.indexOf("://"); + URI uri = URI.create(schemeSeparator > 0 ? endpoint : "https://" + endpoint); + String host = uri.getHost(); if (host == null) { return false; } host = host.toLowerCase(Locale.ROOT); - boolean awsDns = host.endsWith(".amazonaws.com") + return host.endsWith(".amazonaws.com") || host.endsWith(".amazonaws.com.cn") || host.endsWith(".api.aws"); - return awsDns && (host.startsWith("s3.") - || host.startsWith("s3-") - || host.startsWith("s3express-") - || host.contains(".s3.") - || host.contains(".s3-") - || host.contains(".s3express-")); - } - - private static boolean isDirectoryBucketName(String bucketName) { - String suffix = "--x-s3"; - if (bucketName == null || !bucketName.endsWith(suffix)) { - return false; - } - String prefix = bucketName.substring(0, bucketName.length() - suffix.length()); - int zoneSeparator = prefix.lastIndexOf("--"); - return zoneSeparator > 0 && prefix.substring(zoneSeparator + 2).contains("-az"); } @Override From dccf3113a722c4e47c69849519bbfaa14608e6a2 Mon Sep 17 00:00:00 2001 From: Refrain Date: Sat, 18 Jul 2026 20:17:39 +0800 Subject: [PATCH 09/13] [feature](s3) Support S3 Express reads for S3 imports ### What problem does this PR solve? Issue Number: None Related PR: #65504 Problem Summary: Amazon S3 Express One Zone directory buckets require directory-bucket listing rules and SDK-managed session authentication, while the existing S3 import paths used regular S3 clients. Scope the capability to one-shot INSERT SELECT from the S3 table-valued source and Broker Load WITH S3, validate native AWS directory-bucket context, select Express clients for discovery and reads, and preserve ordinary, third-party, legacy, and non-target import behavior. ### Release note Support importing from Amazon S3 Express One Zone directory buckets through one-shot INSERT SELECT S3 sources and Broker Load WITH S3. ### Check List (For Author) - Test: Unit Test - FE core targeted unit tests - FE filesystem targeted unit tests - BE targeted unit tests - Behavior changed: Yes, S3 Express reads are enabled only for the two scoped S3 import entry points. - Does this need documentation: Yes (follow-up documentation PR pending) --- be/src/io/fs/s3_file_system.cpp | 1 + be/src/io/fs/s3_obj_storage_client.cpp | 14 +- be/src/io/fs/s3_obj_storage_client.h | 5 +- be/src/util/s3_util.cpp | 235 +++++++++++++++--- be/src/util/s3_util.h | 40 ++- .../io/fs/s3_obj_stroage_client_mock_test.cpp | 58 ++--- be/test/io/s3_client_factory_test.cpp | 147 ++++++++++- be/test/util/s3_util_test.cpp | 27 +- .../doris/cloud/load/CloudBrokerLoadJob.java | 3 +- .../org/apache/doris/common/util/S3Util.java | 64 ----- .../AbstractS3CompatibleProperties.java | 10 + .../doris/load/loadv2/BrokerLoadJob.java | 22 +- .../load/loadv2/BrokerLoadPendingTask.java | 31 +-- .../doris/nereids/StatementContext.java | 10 + .../trees/expressions/functions/table/S3.java | 6 +- .../org/apache/doris/qe/StmtExecutor.java | 15 ++ .../ExternalFileTableValuedFunction.java | 30 +-- .../tablefunction/S3TableValuedFunction.java | 24 +- .../apache/doris/common/util/S3UtilTest.java | 32 +-- .../property/storage/S3PropertiesTest.java | 15 ++ .../doris/load/loadv2/BrokerLoadJobTest.java | 24 ++ .../doris/qe/S3ExpressImportScopeTest.java | 62 +++++ .../ExternalFileTableValuedFunctionTest.java | 60 +++++ .../doris/filesystem/s3/S3FileSystem.java | 17 +- .../filesystem/s3/S3FileSystemProperties.java | 6 + .../doris/filesystem/s3/S3ObjStorage.java | 48 +++- .../doris/filesystem/s3/S3FileSystemTest.java | 92 +++++++ .../filesystem/s3/S3ObjStorageMockTest.java | 235 +++++++++++++++++- .../spi/S3CompatibleFileSystem.java | 9 +- 29 files changed, 1067 insertions(+), 275 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 63be8f1955a59c..88ef46e61d6e18 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -101,6 +101,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.max_connections = conf.max_connections; reset_conf.request_timeout_ms = conf.request_timeout_ms; reset_conf.use_virtual_addressing = conf.use_virtual_addressing; + reset_conf.enable_s3_express_read = conf.enable_s3_express_read; reset_conf.role_arn = conf.role_arn; reset_conf.external_id = conf.external_id; diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp index 00c2fbd49fd2b1..f9ed8e155ff59c 100644 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ b/be/src/io/fs/s3_obj_storage_client.cpp @@ -164,11 +164,8 @@ ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOpti Aws::S3::Model::PutObjectRequest request; request.WithBucket(opts.bucket).WithKey(opts.key); auto string_view_stream = std::make_shared(stream.data(), stream.size()); - if (!_is_s3_express) { - Aws::Utils::ByteBuffer part_md5( - Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); - } + Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); request.SetBody(string_view_stream); request.SetContentLength(stream.size()); request.SetContentType("application/octet-stream"); @@ -212,11 +209,8 @@ ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStorageP request.SetBody(string_view_stream); - if (!_is_s3_express) { - Aws::Utils::ByteBuffer part_md5( - Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); - } + Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); request.SetContentLength(stream.size()); request.SetContentType("application/octet-stream"); diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h index 770f1e15c4392f..45294226594d81 100644 --- a/be/src/io/fs/s3_obj_storage_client.h +++ b/be/src/io/fs/s3_obj_storage_client.h @@ -32,9 +32,7 @@ class ObjClientHolder; class S3ObjStorageClient final : public ObjStorageClient { public: - explicit S3ObjStorageClient(std::shared_ptr client, - bool is_s3_express = false) - : _client(std::move(client)), _is_s3_express(is_s3_express) {} + S3ObjStorageClient(std::shared_ptr client) : _client(std::move(client)) {} ~S3ObjStorageClient() override = default; ObjectStorageUploadResponse create_multipart_upload( const ObjectStoragePathOptions& opts) override; @@ -60,7 +58,6 @@ class S3ObjStorageClient final : public ObjStorageClient { private: std::shared_ptr _client; - bool _is_s3_express = false; }; } // namespace doris::io diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index b14d4737bb5946..a22bf69ee9f8d7 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,9 @@ #include #include #include +#ifdef BE_TEST +#include +#endif #include #include #include @@ -87,8 +91,8 @@ namespace { std::string ascii_lower(std::string_view value) { std::string result(value); - std::transform(result.begin(), result.end(), result.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); + std::ranges::transform(result, result.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); return result; } @@ -105,16 +109,18 @@ AwsS3Endpoint parse_aws_s3_endpoint(std::string_view endpoint) { if (scheme_separator != std::string_view::npos) { result.https = ascii_lower(endpoint.substr(0, scheme_separator)) == "https"; endpoint.remove_prefix(scheme_separator + 3); + } else { + result.https = true; } const auto path_separator = endpoint.find_first_of("/?#"); const auto authority = endpoint.substr(0, path_separator); - const auto path = path_separator == std::string_view::npos - ? std::string_view {} - : endpoint.substr(path_separator); + const auto path = path_separator == std::string_view::npos ? std::string_view {} + : endpoint.substr(path_separator); bool standard_port = true; auto host = authority; - if (const auto port_separator = authority.rfind(':'); port_separator != std::string_view::npos) { + if (const auto port_separator = authority.rfind(':'); + port_separator != std::string_view::npos) { standard_port = authority.substr(port_separator + 1) == "443"; host = authority.substr(0, port_separator); } @@ -141,8 +147,9 @@ AwsS3Endpoint parse_aws_s3_endpoint(std::string_view endpoint) { !lower_host.starts_with(regional_prefix)) { return result; } - const auto region = std::string_view(lower_host).substr( - regional_prefix.size(), lower_host.size() - regional_prefix.size() - domain.size()); + const auto region = std::string_view(lower_host) + .substr(regional_prefix.size(), + lower_host.size() - regional_prefix.size() - domain.size()); if (region.empty() || region.find('.') != std::string_view::npos) { return result; } @@ -155,6 +162,77 @@ bool has_s3_directory_bucket_suffix(std::string_view bucket) { return bucket.ends_with("--x-s3"); } +class S3CompatibleEndpointProvider final : public Aws::S3::S3EndpointProvider { +public: + explicit S3CompatibleEndpointProvider(std::string_view endpoint) { + std::string normalized_endpoint(endpoint); + if (normalized_endpoint.find("://") == std::string::npos) { + normalized_endpoint.insert(0, "https://"); + } + _endpoint_authority = Aws::Http::URI(normalized_endpoint.c_str()).GetAuthority(); + DORIS_CHECK(!_endpoint_authority.empty()); + } + + Aws::Endpoint::ResolveEndpointOutcome ResolveEndpoint( + const Aws::Endpoint::EndpointParameters& endpoint_parameters) const override { + auto compatible_parameters = endpoint_parameters; + Aws::String bucket; + Aws::String surrogate_bucket; + for (auto& parameter : compatible_parameters) { + Aws::String value; + if (parameter.GetName() == "Bucket" && + parameter.GetString(value) == + Aws::Endpoint::EndpointParameter::GetSetResult::SUCCESS && + has_s3_directory_bucket_suffix(std::string_view(value.data(), value.size()))) { + bucket = std::move(value); + surrogate_bucket = bucket; + // The SDK selects S3 Express endpoint/signing rules solely from this suffix. + // Resolve an otherwise identical bucket through regular S3 rules, then restore + // the real bucket in the resolved URI below. + surrogate_bucket.back() = '2'; + parameter.SetString(surrogate_bucket); + break; + } + } + + auto outcome = Aws::S3::S3EndpointProvider::ResolveEndpoint(compatible_parameters); + if (bucket.empty() || !outcome.IsSuccess()) { + return outcome; + } + + auto& endpoint = outcome.GetResult(); + auto uri = endpoint.GetURI(); + auto authority = uri.GetAuthority(); + auto path = uri.GetPath(); + const auto bucket_pos = path.rfind(surrogate_bucket); + const bool bucket_is_path_segment = bucket_pos != Aws::String::npos && + (bucket_pos == 0 || path[bucket_pos - 1] == '/') && + (bucket_pos + surrogate_bucket.size() == path.size() || + path[bucket_pos + surrogate_bucket.size()] == '/'); + const auto virtual_host_authority = surrogate_bucket + "." + _endpoint_authority; + if (authority == _endpoint_authority) { + DORIS_CHECK(bucket_is_path_segment); + path.replace(bucket_pos, surrogate_bucket.size(), bucket); + uri.SetPath(path); + } else { + DORIS_CHECK(authority == virtual_host_authority); + authority.replace(0, surrogate_bucket.size(), bucket); + uri.SetAuthority(authority); + } + endpoint.SetURI(std::move(uri)); + return outcome; + } + +private: + Aws::String _endpoint_authority; +}; + +bool is_legacy_s3_directory_bucket_endpoint(std::string_view endpoint) { + const auto lower_endpoint = ascii_lower(endpoint); + return lower_endpoint.find("s3express-control.") != std::string::npos || + lower_endpoint.find("s3express-") != std::string::npos; +} + bool is_s3_directory_bucket_name(std::string_view bucket) { constexpr std::string_view suffix = "--x-s3"; if (!bucket.ends_with(suffix)) { @@ -168,7 +246,7 @@ bool is_s3_directory_bucket_name(std::string_view bucket) { const auto base_name = bucket.substr(0, zone_separator); if (!std::isalnum(static_cast(base_name.front())) || !std::isalnum(static_cast(base_name.back())) || - !std::all_of(base_name.begin(), base_name.end(), [](unsigned char c) { + !std::ranges::all_of(base_name, [](unsigned char c) { return std::islower(c) || std::isdigit(c) || c == '-'; })) { return false; @@ -177,10 +255,12 @@ bool is_s3_directory_bucket_name(std::string_view bucket) { const auto az_separator = zone_id.rfind("-az"); return az_separator != std::string_view::npos && az_separator > 0 && az_separator + 3 < zone_id.size() && - std::all_of(zone_id.begin(), zone_id.begin() + az_separator, - [](unsigned char c) { return std::islower(c) || std::isdigit(c) || c == '-'; }) && - std::all_of(zone_id.begin() + az_separator + 3, zone_id.end(), - [](unsigned char c) { return std::isdigit(c); }); + std::ranges::all_of(zone_id.substr(0, az_separator), + [](unsigned char c) { + return std::islower(c) || std::isdigit(c) || c == '-'; + }) && + std::ranges::all_of(zone_id.substr(az_separator + 3), + [](unsigned char c) { return std::isdigit(c); }); } doris::Status is_s3_conf_valid(const S3ClientConf& conf) { @@ -191,7 +271,9 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } const auto endpoint = parse_aws_s3_endpoint(conf.endpoint); - if (endpoint.aws_domain && has_s3_directory_bucket_suffix(conf.bucket)) { + if (conf.enable_s3_express_read && conf.provider == io::ObjStorageType::AWS && + endpoint.aws_domain && has_s3_directory_bucket_suffix(conf.bucket) && + !is_legacy_s3_directory_bucket_endpoint(conf.endpoint)) { if (!is_s3_directory_bucket_name(conf.bucket)) { return Status::InvalidArgument( "Invalid AWS Directory Bucket name {}; expected " @@ -614,13 +696,12 @@ std::shared_ptr S3ClientFactory::get_aws_cred std::shared_ptr S3ClientFactory::_create_s3_client( const S3ClientConf& s3_conf) { - const bool s3_express = - is_s3_express(s3_conf.bucket, s3_conf.endpoint, s3_conf.region); + const auto build_options = get_s3_client_build_options(s3_conf); TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", std::make_shared(std::make_shared())); Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); - if (s3_conf.need_override_endpoint && !s3_express) { + if (build_options.override_endpoint) { aws_config.endpointOverride = s3_conf.endpoint; } aws_config.region = s3_conf.region; @@ -648,7 +729,7 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms; } - if (s3_express) { + if (build_options.force_https) { aws_config.scheme = Aws::Http::Scheme::HTTPS; } else if (config::s3_client_http_scheme == "http") { aws_config.scheme = Aws::Http::Scheme::HTTP; @@ -658,22 +739,24 @@ std::shared_ptr S3ClientFactory::_create_s3_client( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true); std::shared_ptr new_client; - if (s3_express) { - Aws::S3::S3ClientConfiguration express_config( - aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent, - true); - express_config.disableS3ExpressAuth = false; - new_client = std::make_shared( - get_aws_credentials_provider(s3_conf), - Aws::MakeShared("S3Client"), express_config); - } else if (has_s3_directory_bucket_suffix(s3_conf.bucket)) { - Aws::S3::S3ClientConfiguration compatible_config( - aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - s3_conf.use_virtual_addressing); - compatible_config.disableS3ExpressAuth = true; - new_client = std::make_shared( - get_aws_credentials_provider(s3_conf), - Aws::MakeShared("S3Client"), compatible_config); + if (build_options.use_endpoint_provider) { + const auto signing_policy = + build_options.request_dependent_signing + ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent + : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; + Aws::S3::S3ClientConfiguration endpoint_provider_config( + aws_config, signing_policy, build_options.use_virtual_addressing); + endpoint_provider_config.disableS3ExpressAuth = build_options.disable_s3_express_auth; + std::shared_ptr endpoint_provider; + if (build_options.use_compatible_endpoint_provider) { + endpoint_provider = + Aws::MakeShared("S3Client", s3_conf.endpoint); + } else { + endpoint_provider = Aws::MakeShared("S3Client"); + } + new_client = std::make_shared(get_aws_credentials_provider(s3_conf), + std::move(endpoint_provider), + endpoint_provider_config); } else { new_client = std::make_shared( get_aws_credentials_provider(s3_conf), std::move(aws_config), @@ -681,8 +764,7 @@ std::shared_ptr S3ClientFactory::_create_s3_client( s3_conf.use_virtual_addressing); } - auto obj_client = - std::make_shared(std::move(new_client), s3_express); + auto obj_client = std::make_shared(std::move(new_client)); LOG_INFO("create one s3 client with {}", s3_conf.to_string()); return obj_client; } @@ -739,6 +821,10 @@ Status S3ClientFactory::convert_properties_to_s3_conf( s3_conf->bucket = s3_uri.get_bucket(); // For azure's compatibility s3_conf->client_conf.bucket = s3_uri.get_bucket(); + const auto express_import = properties.find(S3_EXPRESS_IMPORT_READ); + s3_conf->client_conf.enable_s3_express_read = express_import != properties.end() && + express_import->second == "true" && + has_s3_directory_bucket_suffix(s3_conf->bucket); s3_conf->prefix = ""; // See https://sdk.amazonaws.com/cpp/api/LATEST/class_aws_1_1_s3_1_1_s3_client.html @@ -760,7 +846,6 @@ Status S3ClientFactory::convert_properties_to_s3_conf( if (auto it = properties.find(S3_CREDENTIALS_PROVIDER_TYPE); it != properties.end()) { s3_conf->client_conf.cred_provider_type = cred_provider_type_from_string(it->second); } - if (auto st = is_s3_conf_valid(s3_conf->client_conf); !st.ok()) { return st; } @@ -950,4 +1035,80 @@ bool is_s3_express(std::string_view bucket, std::string_view endpoint, std::stri parsed_endpoint.region == ascii_lower(region); } +S3ClientBuildOptions get_s3_client_build_options(const S3ClientConf& conf) { + const bool s3_express = conf.enable_s3_express_read && + conf.provider == io::ObjStorageType::AWS && + is_s3_express(conf.bucket, conf.endpoint, conf.region); + if (s3_express) { + return {.use_endpoint_provider = true, + .use_compatible_endpoint_provider = false, + .override_endpoint = false, + .force_https = true, + .use_virtual_addressing = true, + .request_dependent_signing = true, + .disable_s3_express_auth = false}; + } + + const bool compatible_directory_bucket = conf.enable_s3_express_read && + conf.need_override_endpoint && + has_s3_directory_bucket_suffix(conf.bucket) && + !parse_aws_s3_endpoint(conf.endpoint).aws_domain; + if (compatible_directory_bucket) { + return {.use_endpoint_provider = true, + .use_compatible_endpoint_provider = true, + .override_endpoint = conf.need_override_endpoint, + .force_https = false, + .use_virtual_addressing = conf.use_virtual_addressing, + .request_dependent_signing = false, + .disable_s3_express_auth = true}; + } + + return {.use_endpoint_provider = false, + .use_compatible_endpoint_provider = false, + .override_endpoint = conf.need_override_endpoint, + .force_https = false, + .use_virtual_addressing = conf.use_virtual_addressing, + .request_dependent_signing = false, + .disable_s3_express_auth = false}; +} + +#ifdef BE_TEST +S3EndpointResolutionForTest resolve_s3_compatible_endpoint_for_test(std::string_view endpoint, + std::string_view region, + std::string_view bucket, + bool use_virtual_addressing) { + Aws::Client::ClientConfiguration client_config; + client_config.endpointOverride = std::string(endpoint); + client_config.region = std::string(region); + Aws::S3::S3ClientConfiguration s3_config( + client_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + use_virtual_addressing); + s3_config.disableS3ExpressAuth = true; + + S3CompatibleEndpointProvider provider(endpoint); + provider.InitBuiltInParameters(s3_config); + provider.AccessClientContextParameters().SetForcePathStyle(!use_virtual_addressing); + provider.AccessClientContextParameters().SetDisableS3ExpressSessionAuth(true); + provider.OverrideEndpoint(client_config.endpointOverride); + + Aws::S3::Model::GetObjectRequest request; + request.SetBucket(std::string(bucket)); + request.SetKey("object"); + auto outcome = provider.ResolveEndpoint(request.GetEndpointContextParams()); + DORIS_CHECK(outcome.IsSuccess()); + const auto& endpoint_result = outcome.GetResult(); + const auto& attributes = endpoint_result.GetAttributes(); + DORIS_CHECK(attributes.has_value()); + const auto& signing_name = attributes->authScheme.GetSigningName(); + DORIS_CHECK(signing_name.has_value()); + return {.url = endpoint_result.GetURL(), + .authority = endpoint_result.GetURI().GetAuthority(), + .path = endpoint_result.GetURI().GetPath(), + .backend = attributes->backend, + .signing_name = signing_name.value(), + .port = endpoint_result.GetURI().GetPort(), + .use_s3_express_auth = attributes->useS3ExpressAuth}; +} +#endif + } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 25fd530769cd51..1bcc10129ebcbc 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -50,6 +50,8 @@ class Adder; namespace doris { +inline constexpr char S3_EXPRESS_IMPORT_READ[] = "__DORIS_S3_EXPRESS_IMPORT_READ__"; + namespace s3_bvar { extern bvar::LatencyRecorder s3_get_latency; extern bvar::LatencyRecorder s3_put_latency; @@ -87,6 +89,8 @@ struct S3ClientConf { bool use_virtual_addressing = true; // For aws s3, no need to override endpoint bool need_override_endpoint = true; + // Enable S3 Express client selection for a scoped import read. + bool enable_s3_express_read = false; CredProviderType cred_provider_type = CredProviderType::Default; std::string role_arn; @@ -104,6 +108,7 @@ struct S3ClientConf { hash_code ^= request_timeout_ms; hash_code ^= connect_timeout_ms; hash_code ^= use_virtual_addressing; + hash_code ^= enable_s3_express_read ? 0x9e3779b97f4a7c15ULL : 0; hash_code ^= static_cast(provider); hash_code ^= static_cast(cred_provider_type); @@ -116,13 +121,42 @@ struct S3ClientConf { return fmt::format( "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " - "cred_provider_type={},role_arn={}, external_id={}", + "enable_s3_express_read={}, cred_provider_type={},role_arn={}, external_id={}", hide_access_key(ak), token, endpoint, region, bucket, max_connections, - request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, - role_arn, external_id); + request_timeout_ms, connect_timeout_ms, use_virtual_addressing, + enable_s3_express_read, cred_provider_type, role_arn, external_id); } }; +struct S3ClientBuildOptions { + bool use_endpoint_provider = false; + bool use_compatible_endpoint_provider = false; + bool override_endpoint = false; + bool force_https = false; + bool use_virtual_addressing = true; + bool request_dependent_signing = false; + bool disable_s3_express_auth = false; +}; + +S3ClientBuildOptions get_s3_client_build_options(const S3ClientConf& conf); + +#ifdef BE_TEST +struct S3EndpointResolutionForTest { + std::string url; + std::string authority; + std::string path; + std::string backend; + std::string signing_name; + uint16_t port = 0; + bool use_s3_express_auth = false; +}; + +S3EndpointResolutionForTest resolve_s3_compatible_endpoint_for_test(std::string_view endpoint, + std::string_view region, + std::string_view bucket, + bool use_virtual_addressing); +#endif + struct S3Conf { std::string bucket; std::string prefix; diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index e4e39590892a53..a23565af378e0f 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -17,11 +17,11 @@ #include #include +#include +#include #include #include #include -#include -#include #include "gmock/gmock.h" #include "io/fs/s3_obj_storage_client.h" @@ -37,10 +37,8 @@ class MockS3Client : public Aws::S3::S3Client { MOCK_METHOD(Aws::S3::Model::ListObjectsV2Outcome, ListObjectsV2, (const Aws::S3::Model::ListObjectsV2Request& request), (const, override)); - MOCK_METHOD(Aws::S3::Model::PutObjectOutcome, PutObject, - (const Aws::S3::Model::PutObjectRequest& request), (const, override)); - MOCK_METHOD(Aws::S3::Model::UploadPartOutcome, UploadPart, - (const Aws::S3::Model::UploadPartRequest& request), (const, override)); + MOCK_METHOD(Aws::S3::Model::GetObjectOutcome, GetObject, + (const Aws::S3::Model::GetObjectRequest& request), (const, override)); }; class S3ObjStorageClientMockTest : public testing::Test { @@ -126,35 +124,27 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { files.clear(); } -TEST_F(S3ObjStorageClientMockTest, content_md5_depends_on_s3_express) { - const auto verify_requests = [](bool is_s3_express, bool expect_content_md5) { - auto mock_s3_client = std::make_shared(); - S3ObjStorageClient s3_obj_storage_client(mock_s3_client, is_s3_express); - - EXPECT_CALL(*mock_s3_client, PutObject(testing::_)) - .WillOnce([expect_content_md5](const PutObjectRequest& request) { - EXPECT_EQ(expect_content_md5, request.ContentMD5HasBeenSet()); - return PutObjectOutcome(PutObjectResult {}); - }); - auto put_response = s3_obj_storage_client.put_object( - {.bucket = "dummy-bucket", .key = "content-md5-put"}, "put-body"); - EXPECT_EQ(ErrorCode::OK, put_response.status.code); - - EXPECT_CALL(*mock_s3_client, UploadPart(testing::_)) - .WillOnce([expect_content_md5](const UploadPartRequest& request) { - EXPECT_EQ(expect_content_md5, request.ContentMD5HasBeenSet()); - return UploadPartOutcome(UploadPartResult {}); - }); - auto upload_response = s3_obj_storage_client.upload_part( - {.bucket = "dummy-bucket", - .key = "content-md5-multipart", - .upload_id = "upload-id"}, - "part-body", 1); - EXPECT_EQ(ErrorCode::OK, upload_response.resp.status.code); - }; +TEST_F(S3ObjStorageClientMockTest, get_object_sets_range) { + auto mock_s3_client = std::make_shared(); + S3ObjStorageClient s3_obj_storage_client(mock_s3_client); + + EXPECT_CALL(*mock_s3_client, GetObject(testing::_)) + .WillOnce([](const GetObjectRequest& request) { + EXPECT_EQ(request.GetBucket(), "dummy-bucket"); + EXPECT_EQ(request.GetKey(), "data.parquet"); + EXPECT_EQ(request.GetRange(), "bytes=7-10"); + GetObjectResult result; + result.SetContentLength(4); + return GetObjectOutcome(std::move(result)); + }); - verify_requests(false, true); - verify_requests(true, false); + char buffer[4]; + size_t size_return = 0; + auto response = + s3_obj_storage_client.get_object({.bucket = "dummy-bucket", .key = "data.parquet"}, + buffer, 7, sizeof(buffer), &size_return); + EXPECT_EQ(ErrorCode::OK, response.status.code); + EXPECT_EQ(sizeof(buffer), size_return); } TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 2b37161289366b..68c26e58df4e6e 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -30,7 +30,10 @@ namespace doris { class S3ClientFactoryTest : public testing::Test { +protected: FRIEND_TEST(S3ClientFactoryTest, S3ClientFactory); + + static void SetUpTestSuite() { static_cast(S3ClientFactory::instance()); } }; TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { @@ -234,7 +237,7 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfCredentialValidation) { } } -TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ExpressConf) { +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ExpressReadConf) { S3URI s3_uri("s3://analytics--usw2-az1--x-s3/path/to/data.parquet"); ASSERT_TRUE(s3_uri.parse().ok()); @@ -242,18 +245,28 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ExpressConf) { {"AWS_ENDPOINT", "https://s3.us-west-2.amazonaws.com"}, {"AWS_REGION", "us-west-2"}, }; + S3Conf non_read_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &non_read_conf) + .ok()); + ASSERT_FALSE(non_read_conf.client_conf.enable_s3_express_read); + + properties[S3_EXPRESS_IMPORT_READ] = "true"; S3Conf s3_conf; ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + ASSERT_TRUE(s3_conf.client_conf.enable_s3_express_read); + + properties["AWS_ENDPOINT"] = "s3.us-west-2.amazonaws.com"; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); properties["AWS_ENDPOINT"] = "http://s3.us-west-2.amazonaws.com"; ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); properties["AWS_ENDPOINT"] = "https://s3express-control.us-west-2.amazonaws.com"; - ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); properties["AWS_ENDPOINT"] = "https://analytics--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"; - ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); properties["AWS_ENDPOINT"] = "https://s3.us-west-2.amazonaws.com"; properties["AWS_REGION"] = "us-east-1"; @@ -265,6 +278,134 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ExpressConf) { properties["AWS_ENDPOINT"] = "https://minio.example.com"; ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + ASSERT_TRUE(s3_conf.client_conf.enable_s3_express_read); + ASSERT_FALSE(is_s3_express(s3_conf.client_conf.bucket, s3_conf.client_conf.endpoint, + s3_conf.client_conf.region)); + + S3URI ordinary_uri("s3://ordinary-bucket/path/to/data.parquet"); + ASSERT_TRUE(ordinary_uri.parse().ok()); + properties["AWS_ENDPOINT"] = "https://s3.us-west-2.amazonaws.com"; + properties.erase("use_path_style"); + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, ordinary_uri, &s3_conf) + .ok()); + ASSERT_FALSE(s3_conf.client_conf.enable_s3_express_read); +} + +TEST_F(S3ClientFactoryTest, S3ExpressReadIsPartOfClientCacheKey) { + S3ClientConf regular_conf; + regular_conf.bucket = "analytics--usw2-az1--x-s3"; + regular_conf.endpoint = "s3.us-west-2.amazonaws.com"; + regular_conf.region = "us-west-2"; + + S3ClientConf express_read_conf = regular_conf; + express_read_conf.enable_s3_express_read = true; + + ASSERT_NE(regular_conf.get_hash(), express_read_conf.get_hash()); +} + +TEST_F(S3ClientFactoryTest, S3ExpressReadClientBuildOptions) { + S3ClientConf conf; + conf.bucket = "analytics--usw2-az1--x-s3"; + conf.endpoint = "https://s3.us-west-2.amazonaws.com"; + conf.region = "us-west-2"; + conf.enable_s3_express_read = true; + + auto options = get_s3_client_build_options(conf); + EXPECT_TRUE(options.use_endpoint_provider); + EXPECT_FALSE(options.use_compatible_endpoint_provider); + EXPECT_FALSE(options.override_endpoint); + EXPECT_TRUE(options.force_https); + EXPECT_TRUE(options.use_virtual_addressing); + EXPECT_TRUE(options.request_dependent_signing); + EXPECT_FALSE(options.disable_s3_express_auth); + + conf.endpoint = "https://minio.example.com"; + conf.use_virtual_addressing = false; + options = get_s3_client_build_options(conf); + EXPECT_TRUE(options.use_endpoint_provider); + EXPECT_TRUE(options.use_compatible_endpoint_provider); + EXPECT_TRUE(options.override_endpoint); + EXPECT_FALSE(options.force_https); + EXPECT_FALSE(options.use_virtual_addressing); + EXPECT_FALSE(options.request_dependent_signing); + EXPECT_TRUE(options.disable_s3_express_auth); + + conf.need_override_endpoint = false; + options = get_s3_client_build_options(conf); + EXPECT_FALSE(options.use_endpoint_provider); + EXPECT_FALSE(options.use_compatible_endpoint_provider); + EXPECT_FALSE(options.override_endpoint); + EXPECT_FALSE(options.disable_s3_express_auth); + conf.need_override_endpoint = true; + + conf.endpoint = "https://s3express-control.us-west-2.amazonaws.com"; + conf.use_virtual_addressing = true; + options = get_s3_client_build_options(conf); + EXPECT_FALSE(options.use_endpoint_provider); + EXPECT_FALSE(options.use_compatible_endpoint_provider); + EXPECT_TRUE(options.override_endpoint); + EXPECT_FALSE(options.force_https); + EXPECT_TRUE(options.use_virtual_addressing); + EXPECT_FALSE(options.request_dependent_signing); + EXPECT_FALSE(options.disable_s3_express_auth); + + conf.bucket = "ordinary-bucket"; + conf.endpoint = "https://s3.us-west-2.amazonaws.com"; + options = get_s3_client_build_options(conf); + EXPECT_FALSE(options.use_endpoint_provider); + EXPECT_FALSE(options.use_compatible_endpoint_provider); + EXPECT_TRUE(options.override_endpoint); + EXPECT_FALSE(options.force_https); + EXPECT_TRUE(options.use_virtual_addressing); + EXPECT_FALSE(options.request_dependent_signing); + EXPECT_FALSE(options.disable_s3_express_auth); +} + +TEST_F(S3ClientFactoryTest, S3CompatibleDirectoryBucketUsesRegularEndpointRules) { + constexpr std::string_view bucket = "analytics--usw2-az1--x-s3"; + + auto virtual_host = resolve_s3_compatible_endpoint_for_test("https://minio.example.com", + "us-west-2", bucket, true); + EXPECT_EQ(virtual_host.authority, fmt::format("{}.minio.example.com", bucket)); + EXPECT_EQ(virtual_host.signing_name, "s3"); + EXPECT_TRUE(virtual_host.backend.empty()); + EXPECT_FALSE(virtual_host.use_s3_express_auth); + EXPECT_EQ(virtual_host.url.find("--x-s2"), std::string::npos); + + auto path_style = resolve_s3_compatible_endpoint_for_test("https://minio.example.com/base", + "us-west-2", bucket, false); + EXPECT_EQ(path_style.authority, "minio.example.com"); + EXPECT_EQ(path_style.path, fmt::format("/base/{}", bucket)); + EXPECT_EQ(path_style.signing_name, "s3"); + EXPECT_TRUE(path_style.backend.empty()); + EXPECT_FALSE(path_style.use_s3_express_auth); + + auto ip_endpoint = resolve_s3_compatible_endpoint_for_test("http://127.0.0.1:9000", "us-west-2", + bucket, true); + EXPECT_EQ(ip_endpoint.authority, "127.0.0.1"); + EXPECT_EQ(ip_endpoint.port, 9000); + EXPECT_TRUE(ip_endpoint.path.ends_with(fmt::format("/{}", bucket))); + EXPECT_EQ(ip_endpoint.signing_name, "s3"); + EXPECT_TRUE(ip_endpoint.backend.empty()); + EXPECT_FALSE(ip_endpoint.use_s3_express_auth); + + constexpr std::string_view surrogate_bucket = "analytics--usw2-az1--x-s2"; + auto colliding_path_style = resolve_s3_compatible_endpoint_for_test( + fmt::format("https://{}.example.com/base", surrogate_bucket), "us-west-2", bucket, + false); + EXPECT_EQ(colliding_path_style.authority, fmt::format("{}.example.com", surrogate_bucket)); + EXPECT_EQ(colliding_path_style.path, fmt::format("/base/{}", bucket)); + + constexpr std::string_view dotted_bucket = "a.b--x-s3"; + constexpr std::string_view dotted_surrogate_bucket = "a.b--x-s2"; + auto auto_path_style = resolve_s3_compatible_endpoint_for_test( + fmt::format("https://{}.example.com/base", dotted_surrogate_bucket), "us-west-2", + dotted_bucket, true); + EXPECT_EQ(auto_path_style.authority, fmt::format("{}.example.com", dotted_surrogate_bucket)); + EXPECT_EQ(auto_path_style.path, fmt::format("/base/{}", dotted_bucket)); + EXPECT_EQ(auto_path_style.signing_name, "s3"); + EXPECT_TRUE(auto_path_style.backend.empty()); + EXPECT_FALSE(auto_path_style.use_s3_express_auth); } TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) { diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index 45e9030fec52fa..0fe480df0dde07 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -78,28 +78,27 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { } TEST_F(S3UTILTest, is_s3_express_context) { - EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", - "https://s3.us-east-1.amazonaws.com", "us-east-1")); - EXPECT_TRUE(is_s3_express("bucket--cnn1-az1--x-s3", - "https://s3.cn-north-1.amazonaws.com.cn", "cn-north-1")); + EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com", + "us-east-1")); + EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "s3.us-east-1.amazonaws.com", "us-east-1")); + EXPECT_TRUE(is_s3_express("bucket--cnn1-az1--x-s3", "https://s3.cn-north-1.amazonaws.com.cn", + "cn-north-1")); // S3-compatible services must retain their existing endpoint and checksum behavior, // even if a bucket happens to use the Directory Bucket suffix. - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com", - "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com", "us-east-1")); EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com.example.com", "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--x-s3", "https://s3.us-east-1.amazonaws.com", - "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--zone--x-s3", "https://s3.us-east-1.amazonaws.com", - "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--x-s3", "https://s3.us-east-1.amazonaws.com", "us-east-1")); + EXPECT_FALSE( + is_s3_express("bucket--zone--x-s3", "https://s3.us-east-1.amazonaws.com", "us-east-1")); EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path", "us-east-1")); // Users configure the standard HTTPS regional endpoint. The SDK derives the zonal endpoint. - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", - "http://s3.us-east-1.amazonaws.com", "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", - "https://s3.us-east-1.amazonaws.com", "us-west-2")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "http://s3.us-east-1.amazonaws.com", + "us-east-1")); + EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com", + "us-west-2")); EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://s3express-control.us-east-1.amazonaws.com", "us-east-1")); EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudBrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudBrokerLoadJob.java index cd5c8dd1b0c52f..634b8a0b98f71f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudBrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/load/CloudBrokerLoadJob.java @@ -155,7 +155,8 @@ protected LoadLoadingTask createTask(Database db, OlapTable table, List----x-s3"); - } - if (!awsEndpoint.group(1).equalsIgnoreCase(region)) { - throw new UserException("AWS Directory Bucket endpoint region " + awsEndpoint.group(1) - + " does not match configured region " + region); - } - if (usePathStyle) { - throw new UserException("Path-style addressing is not supported for AWS Directory Bucket"); - } - if (Strings.isNullOrEmpty(uri.getKey()) || !getLongestPrefix(path).equals(path) - || uri.getKey().endsWith("/") - || uri.getQueryParams().isPresent()) { - throw new UserException("AWS Directory Bucket TVF and Load paths must identify one complete " - + "object key and cannot contain glob expressions or a directory prefix"); - } - return true; - } - - private static boolean isDirectoryBucketName(String bucketName) { - return bucketName != null && DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches(); - } - - private static boolean isAwsEndpoint(String endpoint) { - int schemeSeparator = endpoint.indexOf("://"); - URI uri = URI.create(schemeSeparator > 0 ? endpoint : "https://" + endpoint); - String host = uri.getHost(); - if (host == null) { - return false; - } - host = host.toLowerCase(Locale.ROOT); - return host.endsWith(".amazonaws.com") - || host.endsWith(".amazonaws.com.cn") - || host.endsWith(".api.aws"); - } - // Apply some rules to extend the globs parsing behavior public static String extendGlobs(String pathPattern) { return extendGlobNumberRange(pathPattern); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index 3b589daab34775..f2de61b671d912 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -54,6 +54,9 @@ */ public abstract class AbstractS3CompatibleProperties extends StorageProperties implements ObjectStorageProperties { private static final Logger LOG = LogManager.getLogger(AbstractS3CompatibleProperties.class); + // Internal FE-to-BE scope marker; raw user properties never bind this flag. + public static final String S3_EXPRESS_IMPORT_READ = "__DORIS_S3_EXPRESS_IMPORT_READ__"; + private boolean s3ExpressImportRead; /** * Constructor to initialize the object storage properties with the provided type and original properties map. @@ -118,9 +121,16 @@ private Map doBuildS3Configuration(String maxConnections, if (StringUtils.isNotBlank(credentialsProviderType)) { s3Props.put("AWS_CREDENTIALS_PROVIDER_TYPE", credentialsProviderType); } + if (s3ExpressImportRead) { + s3Props.put(S3_EXPRESS_IMPORT_READ, Boolean.TRUE.toString()); + } return s3Props; } + public void enableS3ExpressImportRead() { + s3ExpressImportRead = true; + } + protected String getAwsCredentialsProviderTypeForBackend() { if (StringUtils.isBlank(getAccessKey()) && StringUtils.isBlank(getSecretKey())) { return AwsCredentialsProviderMode.ANONYMOUS.name(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index e6092fc94b1e30..f94ec501b52669 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -45,6 +45,7 @@ import org.apache.doris.common.util.MetaLockUtils; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.load.BrokerFileGroup; import org.apache.doris.load.BrokerFileGroupAggInfo.FileGroupAggKey; @@ -69,6 +70,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -125,6 +127,21 @@ public BrokerLoadJob(EtlJobType type, long dbId, String label, BrokerDesc broker } } + protected BrokerDesc brokerDescForS3ExpressImport() { + if (jobType != EtlJobType.BROKER || brokerDesc == null + || brokerDesc.getStorageType() != StorageBackend.StorageType.S3) { + return brokerDesc; + } + Map properties = new HashMap<>(brokerDesc.getProperties()); + BrokerDesc importBrokerDesc = new BrokerDesc( + brokerDesc.getName(), brokerDesc.getStorageType(), properties); + if (importBrokerDesc.getStorageProperties() instanceof AbstractS3CompatibleProperties) { + ((AbstractS3CompatibleProperties) importBrokerDesc.getStorageProperties()) + .enableS3ExpressImportRead(); + } + return importBrokerDesc; + } + @Override public void beginTxn() throws LabelAlreadyUsedException, BeginTransactionException, AnalysisException, DuplicatedRequestException, @@ -146,7 +163,8 @@ protected void unprotectedExecuteJob() { } protected LoadTask createPendingTask() { - return new BrokerLoadPendingTask(this, fileGroupAggInfo.getAggKeyToFileGroups(), brokerDesc, getPriority()); + return new BrokerLoadPendingTask(this, fileGroupAggInfo.getAggKeyToFileGroups(), + brokerDescForS3ExpressImport(), getPriority()); } /** @@ -259,7 +277,7 @@ public void setComputeGroup() { protected LoadLoadingTask createTask(Database db, OlapTable table, List brokerFileGroups, boolean isEnableMemtableOnSinkNode, int batchSize, FileGroupAggKey aggKey, BrokerPendingTaskAttachment attachment) throws UserException { - LoadLoadingTask task = new LoadLoadingTask(this.userInfo, db, table, brokerDesc, + LoadLoadingTask task = new LoadLoadingTask(this.userInfo, db, table, brokerDescForS3ExpressImport(), brokerFileGroups, getDeadlineMs(), getExecMemLimit(), isStrictMode(), isPartialUpdate(), getPartialUpdateNewKeyPolicy(), transactionId, this, getTimeZone(), getTimeout(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java index 98a176e25041ec..c2b9be5db9fccd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadPendingTask.java @@ -24,9 +24,6 @@ import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.common.util.LogBuilder; import org.apache.doris.common.util.LogKey; -import org.apache.doris.common.util.S3Util; -import org.apache.doris.datasource.property.storage.ObjectStorageProperties; -import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.filesystem.FileEntry; import org.apache.doris.filesystem.FileSystem; import org.apache.doris.filesystem.GlobListing; @@ -99,7 +96,6 @@ protected void getAllFileStatus() throws UserException { fileStatusList.add(fileStatuses); } } else { - StorageProperties storageProperties = brokerDesc.getStorageProperties(); for (BrokerFileGroup fileGroup : fileGroups) { long groupFileSize = 0; List fileStatuses = Lists.newArrayList(); @@ -110,20 +106,12 @@ protected void getAllFileStatus() throws UserException { // Plain listFiles uses S3 prefix matching which can return unintended // prefix-siblings (e.g. "file.csv.bz2" when listing "file.csv"). List entries; - if (storageProperties instanceof ObjectStorageProperties - && isExactS3ExpressObject( - path, (ObjectStorageProperties) storageProperties)) { - Location location = Location.of(path); - entries = List.of(new FileEntry( - location, fs.newInputFile(location).length(), false, 0L, List.of())); - } else { - try { - GlobListing listing = fs.globListWithLimit( - Location.of(path), null, 0, 0); - entries = listing.getFiles(); - } catch (UnsupportedOperationException ex) { - entries = fs.listFiles(Location.of(path)); - } + try { + GlobListing listing = fs.globListWithLimit( + Location.of(path), null, 0, 0); + entries = listing.getFiles(); + } catch (UnsupportedOperationException ex) { + entries = fs.listFiles(Location.of(path)); } for (FileEntry e : entries) { fileStatuses.add(new TBrokerFileStatus( @@ -186,11 +174,4 @@ && isExactS3ExpressObject( ((BrokerLoadJob) callback).setLoadFileInfo(totalFileNum, totalFileSize); } - - private static boolean isExactS3ExpressObject(String path, ObjectStorageProperties properties) - throws UserException { - return S3Util.isExactS3ExpressObject( - path, properties.getEndpoint(), properties.getRegion(), - Boolean.parseBoolean(properties.getUsePathStyle())); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index cd6a985218dd60..7e4e134584b883 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -314,6 +314,8 @@ public enum TableFrom { private final Set> materializationRewrittenSuccessSet = new HashSet<>(); private boolean isInsert = false; + // Trusted statement-scope marker for one-shot INSERT SELECT from the internal S3 TVF. + private boolean s3ExpressImportRead = false; private Optional>> mvRefreshPredicates = Optional.empty(); // For Iceberg rewrite operations: store file scan tasks to be used by @@ -1244,6 +1246,14 @@ public boolean isInsert() { return isInsert; } + public void setS3ExpressImportRead(boolean s3ExpressImportRead) { + this.s3ExpressImportRead = s3ExpressImportRead; + } + + public boolean isS3ExpressImportRead() { + return s3ExpressImportRead; + } + public boolean isQueryStatsRecorded() { return queryStatsRecorded; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java index 3c4c924d7f41e6..f650a6b3743420 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/S3.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumn; import org.apache.doris.nereids.trees.plans.logical.SupportPruneNestedColumnFormats; import org.apache.doris.nereids.types.coercion.AnyDataType; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.tablefunction.S3TableValuedFunction; import org.apache.doris.tablefunction.TableValuedFunctionIf; @@ -44,7 +45,10 @@ public FunctionSignature customSignature() { protected TableValuedFunctionIf toCatalogFunction() { try { Map arguments = getTVFProperties().getMap(); - return new S3TableValuedFunction(arguments); + ConnectContext context = ConnectContext.get(); + boolean s3ExpressImportRead = context != null && context.getStatementContext() != null + && context.getStatementContext().isS3ExpressImportRead(); + return new S3TableValuedFunction(arguments, s3ExpressImportRead); } catch (Throwable t) { throw new AnalysisException("Can not build s3(): " + t.getMessage(), t); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 795a81c8a76ed7..edf3c794a68bf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -91,6 +91,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner; import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.Command; @@ -98,6 +99,7 @@ import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand; import org.apache.doris.nereids.trees.plans.commands.DeleteFromUsingCommand; import org.apache.doris.nereids.trees.plans.commands.EmptyCommand; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.Forward; import org.apache.doris.nereids.trees.plans.commands.LoadCommand; import org.apache.doris.nereids.trees.plans.commands.PrepareCommand; @@ -774,6 +776,8 @@ private void executeByNereids(TUniqueId queryId) throws Exception { "Nereids only process LogicalPlanAdapter, but parsedStmt is " + parsedStmt.getClass().getName()); context.getState().setNereids(true); LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + context.getStatementContext().setS3ExpressImportRead( + shouldEnableS3ExpressImportRead(logicalPlan, context.getState().isInternal())); checkSqlBlocked(logicalPlan.getClass()); if (context.getCommand() == MysqlCommand.COM_STMT_PREPARE) { if (isForwardToMaster()) { @@ -976,6 +980,17 @@ private void executeByNereids(TUniqueId queryId) throws Exception { } } + static boolean shouldEnableS3ExpressImportRead(LogicalPlan logicalPlan, boolean internal) { + if (internal) { + return false; + } + if (logicalPlan instanceof ExplainCommand) { + logicalPlan = ((ExplainCommand) logicalPlan).getLogicalPlan(); + } + return logicalPlan instanceof InsertIntoTableCommand + && logicalPlan.getType() == PlanType.INSERT_INTO_TABLE_COMMAND; + } + public static void initBlockSqlAstNames() { blockSqlAstNames.clear(); blockSqlAstNames = Pattern.compile(",") diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 5d673365cc96b5..6c7962e7267033 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -166,21 +166,14 @@ protected void parseFile() throws AnalysisException { } try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(brokerDesc)) { List entries; - if (sp instanceof ObjectStorageProperties - && isExactS3ExpressObject(path, (ObjectStorageProperties) sp)) { - Location location = Location.of(path); - entries = List.of(new FileEntry( - location, fs.newInputFile(location).length(), false, 0L, List.of())); - } else { - // Always prefer glob semantics: for exact paths it ensures precise matching - // (prevents S3 prefix-based listing from including unintended files like - // "file.csv.bz2" when listing "file.csv"). Fall back to listFiles only - // when the filesystem does not support glob. - try { - entries = fs.globListWithLimit(Location.of(path), "", 0, 0).getFiles(); - } catch (UnsupportedOperationException ex) { - entries = fs.listFiles(Location.of(path)); - } + // Always prefer glob semantics: for exact paths it ensures precise matching + // (prevents S3 prefix-based listing from including unintended files like + // "file.csv.bz2" when listing "file.csv"). Fall back to listFiles only + // when the filesystem does not support glob. + try { + entries = fs.globListWithLimit(Location.of(path), "", 0, 0).getFiles(); + } catch (UnsupportedOperationException ex) { + entries = fs.listFiles(Location.of(path)); } for (FileEntry e : entries) { fileStatuses.add(new TBrokerFileStatus( @@ -199,13 +192,6 @@ && isExactS3ExpressObject(path, (ObjectStorageProperties) sp)) { } } - private static boolean isExactS3ExpressObject(String path, ObjectStorageProperties properties) - throws UserException { - return S3Util.isExactS3ExpressObject( - path, properties.getEndpoint(), properties.getRegion(), - Boolean.parseBoolean(properties.getUsePathStyle())); - } - // The keys in properties map need to be lowercase. protected Map parseCommonProperties(Map properties) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 7d38c50f565ac2..c21568eb2cf283 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TFileType; @@ -39,12 +40,22 @@ */ public class S3TableValuedFunction extends ExternalFileTableValuedFunction { public static final String NAME = "s3"; + private final boolean s3ExpressImportRead; public S3TableValuedFunction(Map properties) throws AnalysisException { + this(properties, false); + } + + public S3TableValuedFunction(Map properties, boolean s3ExpressImportRead) + throws AnalysisException { + this.s3ExpressImportRead = s3ExpressImportRead; // 1. analyze common properties Map props = super.parseCommonProperties(properties); try { this.storageProperties = StorageProperties.createPrimary(props); + if (s3ExpressImportRead) { + enableS3ExpressImportRead(storageProperties); + } this.backendConnectProperties.putAll(storageProperties.getBackendConfigProperties()); String uri = storageProperties.validateAndGetUri(props); filePath = storageProperties.validateAndNormalizeUri(uri); @@ -76,7 +87,17 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - return new BrokerDesc("S3TvfBroker", processedParams); + BrokerDesc brokerDesc = new BrokerDesc("S3TvfBroker", processedParams); + if (s3ExpressImportRead) { + enableS3ExpressImportRead(brokerDesc.getStorageProperties()); + } + return brokerDesc; + } + + private static void enableS3ExpressImportRead(StorageProperties properties) { + if (properties instanceof AbstractS3CompatibleProperties) { + ((AbstractS3CompatibleProperties) properties).enableS3ExpressImportRead(); + } } // =========== implement abstract methods of TableValuedFunctionIf ================= @@ -85,4 +106,3 @@ public String getTableName() { return "S3TableValuedFunction"; } } - diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java index 41b374fea0bb9e..4b976ed86cd3c5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java @@ -17,8 +17,6 @@ package org.apache.doris.common.util; -import org.apache.doris.common.UserException; - import org.junit.Assert; import org.junit.Test; @@ -27,35 +25,6 @@ public class S3UtilTest { - @Test - public void testIsExactS3ExpressObject() throws Exception { - String object = "s3://analytics--use1-az4--x-s3/data/file.parquet"; - String awsEndpoint = "https://s3.us-east-1.amazonaws.com"; - - Assert.assertTrue(S3Util.isExactS3ExpressObject( - object, awsEndpoint, "us-east-1", false)); - Assert.assertFalse(S3Util.isExactS3ExpressObject( - object, "https://minio.example.com", "us-east-1", false)); - Assert.assertFalse(S3Util.isExactS3ExpressObject( - "s3://analytics/data/file.parquet", awsEndpoint, "us-east-1", false)); - - Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( - "s3://analytics--use1-az4--x-s3/data/*.parquet", - awsEndpoint, "us-east-1", false)); - Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( - "s3://analytics--use1-az4--x-s3/data/", - awsEndpoint, "us-east-1", false)); - Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( - "s3://analytics--use1-az4--x-s3", - awsEndpoint, "us-east-1", false)); - Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( - object, "https://s3express-control.us-east-1.amazonaws.com", "us-east-1", false)); - Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( - object, awsEndpoint, "us-west-2", false)); - Assert.assertThrows(UserException.class, () -> S3Util.isExactS3ExpressObject( - object, awsEndpoint, "us-east-1", true)); - } - @Test public void testExtendGlobNumberRange_simpleRange() { // Test simple range expansion {1..3} @@ -488,3 +457,4 @@ public void testExpandBracketPatterns_malformedBracket() { Assert.assertEquals("file[abc.csv", S3Util.expandBracketPatterns("file[abc.csv")); } } + diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index f9cfde32b0633d..edd45643ba6db0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -554,4 +554,19 @@ public void testS3DisableHadoopCache() throws UserException { Assertions.assertEquals("false", s3Properties.hadoopStorageConfig.get("fs.s3.impl.disable.cache")); Assertions.assertEquals("false", s3Properties.hadoopStorageConfig.get("fs.s3n.impl.disable.cache")); } + + @Test + public void testS3ExpressImportMarkerRequiresInternalOptIn() { + origProps.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com"); + origProps.put("s3.region", "us-west-2"); + origProps.put(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ, "true"); + S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); + + Assertions.assertFalse(s3Properties.getBackendConfigProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + + s3Properties.enableS3ExpressImportRead(); + Assertions.assertEquals("true", s3Properties.getBackendConfigProperties() + .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java index 3764e0e286da95..a304be9ff8744d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java @@ -18,6 +18,7 @@ package org.apache.doris.load.loadv2; import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.analysis.StorageBackend; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.OlapTable; @@ -27,9 +28,11 @@ import org.apache.doris.common.Status; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.load.BrokerFileGroup; import org.apache.doris.load.BrokerFileGroupAggInfo; import org.apache.doris.load.BrokerFileGroupAggInfo.FileGroupAggKey; +import org.apache.doris.load.EtlJobType; import org.apache.doris.load.EtlStatus; import org.apache.doris.metric.MetricRepo; import org.apache.doris.nereids.load.NereidsBrokerFileGroup; @@ -67,6 +70,27 @@ public static void start() { MetricRepo.init(); } + @Test + public void testS3ExpressImportPropertyIsTaskScopedToBrokerLoad() { + Map properties = Maps.newHashMap(); + properties.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com"); + properties.put("s3.region", "us-west-2"); + BrokerDesc original = new BrokerDesc("S3", StorageBackend.StorageType.S3, properties); + BrokerLoadJob brokerLoadJob = new BrokerLoadJob(); + Deencapsulation.setField(brokerLoadJob, "brokerDesc", original); + + BrokerDesc taskBrokerDesc = brokerLoadJob.brokerDescForS3ExpressImport(); + + Assert.assertNotSame(original, taskBrokerDesc); + Assert.assertFalse(original.getBackendConfigProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertEquals("true", taskBrokerDesc.getBackendConfigProperties() + .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + + Deencapsulation.setField(brokerLoadJob, "jobType", EtlJobType.COPY); + Assert.assertSame(original, brokerLoadJob.brokerDescForS3ExpressImport()); + } + @Test public void testGetTableNames() throws MetaNotFoundException { BrokerFileGroupAggInfo fileGroupAggInfo = Mockito.mock(BrokerFileGroupAggInfo.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java new file mode 100644 index 00000000000000..7147b65e0e3ba0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.qe; + +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Optional; + +class S3ExpressImportScopeTest { + + @Test + void enableOnlyForTopLevelOneShotInsertIntoTable() { + InsertIntoTableCommand insert = Mockito.mock(InsertIntoTableCommand.class); + Mockito.when(insert.getType()).thenReturn(PlanType.INSERT_INTO_TABLE_COMMAND); + + Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead(insert, false)); + Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead( + new ExplainCommand(ExplainLevel.NORMAL, insert, false), false)); + + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead(insert, true)); + InsertOverwriteTableCommand overwrite = new InsertOverwriteTableCommand( + plan(PlanType.LOGICAL_RESULT_SINK), Optional.empty(), Optional.empty(), Optional.empty()); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + overwrite, false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + new ExplainCommand(ExplainLevel.NORMAL, overwrite, false), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + plan(PlanType.INSERT_INTO_TVF_COMMAND), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + plan(PlanType.LOGICAL_RESULT_SINK), false)); + } + + private static LogicalPlan plan(PlanType type) { + LogicalPlan plan = Mockito.mock(LogicalPlan.class); + Mockito.when(plan.getType()).thenReturn(type); + return plan; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index e5b06bd5dd4101..dd7f5cca53a1d7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -21,8 +21,15 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.FileFormatUtils; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.functions.table.S3; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -116,4 +123,57 @@ public void testCsvSchemaParse() { Assert.fail(); } } + + @Test + public void testS3ExpressMarkerIsScopedToOneShotInsertStatement() throws AnalysisException { + boolean previousRunningUnitTest = FeConstants.runningUnitTest; + ConnectContext previousContext = ConnectContext.get(); + FeConstants.runningUnitTest = true; + try { + Map properties = Maps.newHashMap(); + properties.put("uri", "s3://analytics--usw2-az1--x-s3/data/file.csv"); + properties.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com"); + properties.put("s3.region", "us-west-2"); + properties.put("format", "csv"); + + S3TableValuedFunction directTvf = new S3TableValuedFunction(properties); + Assert.assertFalse(directTvf.getBackendConnectProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertFalse(directTvf.getBrokerDesc().getBackendConfigProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + + ConnectContext context = new ConnectContext(); + StatementContext statementContext = new StatementContext( + context, new OriginStatement("insert into t select * from s3(...)", 0)); + context.setStatementContext(statementContext); + context.setThreadLocalInfo(); + + S3TableValuedFunction queryTvf = (S3TableValuedFunction) new S3(new Properties(properties)) + .getCatalogFunction(); + Assert.assertFalse(queryTvf.getBackendConnectProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + + statementContext.setS3ExpressImportRead(true); + S3TableValuedFunction insertTvf = (S3TableValuedFunction) new S3(new Properties(properties)) + .getCatalogFunction(); + Assert.assertEquals("true", insertTvf.getBackendConnectProperties() + .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertEquals("true", insertTvf.getBrokerDesc().getBackendConfigProperties() + .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertFalse(insertTvf.processedParams + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + + statementContext.setS3ExpressImportRead(false); + S3TableValuedFunction streamingTvf = (S3TableValuedFunction) new S3(new Properties(properties)) + .getCatalogFunction(); + Assert.assertFalse(streamingTvf.getBackendConnectProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + } finally { + FeConstants.runningUnitTest = previousRunningUnitTest; + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } + } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java index f738653ebec568..e1b1ea98eb6a33 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java @@ -19,6 +19,7 @@ import org.apache.doris.filesystem.spi.S3CompatibleFileSystem; +import java.io.IOException; import java.util.List; import java.util.Optional; @@ -28,6 +29,7 @@ public class S3FileSystem extends S3CompatibleFileSystem { private final S3FileSystemProperties properties; + private final S3ObjStorage s3ObjStorage; public S3FileSystem(S3FileSystemProperties properties) { this(properties, new S3ObjStorage(properties)); @@ -36,11 +38,13 @@ public S3FileSystem(S3FileSystemProperties properties) { S3FileSystem(S3FileSystemProperties properties, S3ObjStorage objStorage) { super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); this.properties = properties; + this.s3ObjStorage = objStorage; } public S3FileSystem(S3ObjStorage objStorage) { super(objStorage, objStorage.isUsePathStyle(), objStorage.getSupportedSchemes()); this.properties = null; + this.s3ObjStorage = objStorage; } public Optional properties() { @@ -48,19 +52,20 @@ public Optional properties() { } @Override - protected String globListPrefix(String globPattern) { - if (isDirectoryBucketEndpoint()) { + protected String globListPrefix(String bucket, String globPattern) throws IOException { + if (isDirectoryBucketEndpoint() || s3ObjStorage.usesS3ExpressRead(bucket)) { return slashTerminatedNonGlobPrefix(globPattern); } - return super.globListPrefix(globPattern); + return super.globListPrefix(bucket, globPattern); } @Override - protected List globListPrefixes(String globPattern, String listPrefix) { - if (isDirectoryBucketEndpoint()) { + protected List globListPrefixes(String bucket, String globPattern, String listPrefix) + throws IOException { + if (isDirectoryBucketEndpoint() || s3ObjStorage.usesS3ExpressRead(bucket)) { return List.of(listPrefix); } - return super.globListPrefixes(globPattern, listPrefix); + return super.globListPrefixes(bucket, globPattern, listPrefix); } private boolean isDirectoryBucketEndpoint() { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java index 97cfbb2c4b03d0..da0d0752ac3e7b 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java @@ -63,6 +63,8 @@ public final class S3FileSystemProperties public static final String CONNECTION_TIMEOUT_MS = "s3.connection.timeout"; public static final String USE_PATH_STYLE = "use_path_style"; public static final String CREDENTIALS_PROVIDER_TYPE = "s3.credentials_provider_type"; + // Must match AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ and the BE property key. + static final String S3_EXPRESS_IMPORT_READ = "__DORIS_S3_EXPRESS_IMPORT_READ__"; public static final String DEFAULT_MAX_CONNECTIONS = "50"; public static final String DEFAULT_REQUEST_TIMEOUT_MS = "3000"; @@ -338,6 +340,10 @@ public boolean isDirectoryBucketEndpoint() { || StringUtils.containsIgnoreCase(endpoint, "s3express-"); } + boolean isS3ExpressImportReadEnabled() { + return Boolean.parseBoolean(rawProperties.get(S3_EXPRESS_IMPORT_READ)); + } + private static void putIfNotBlank(Map map, String key, String value) { if (StringUtils.isNotBlank(value)) { map.put(key, value); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index 3f3dce118da8fa..8dc186c8be9d46 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -98,7 +98,7 @@ public class S3ObjStorage implements ObjStorage { private static final Pattern DIRECTORY_BUCKET_PATTERN = Pattern.compile("^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9-]+-az[0-9]+--x-s3$"); private static final Pattern AWS_S3_REGIONAL_ENDPOINT_PATTERN = Pattern.compile( - "^https://s3\\.([a-z0-9-]+)\\.amazonaws\\.com(?:\\.cn)?(?::443)?/?$", + "^(?:https://)?s3\\.([a-z0-9-]+)\\.amazonaws\\.com(?:\\.cn)?(?::443)?/?$", Pattern.CASE_INSENSITIVE); /** Validity period for pre-signed URLs and STS tokens (seconds). */ @@ -173,14 +173,16 @@ private S3Client buildClient(String endpointStr, String region, .httpClient(UrlConnectionHttpClient.builder() .socketTimeout(Duration.ofSeconds(30)) .connectionTimeout(Duration.ofSeconds(30)) - .build()) + .build()) .credentialsProvider(clientCredentialsProvider) .region(Region.of(region)) - .disableS3ExpressSessionAuth(!express) .serviceConfiguration(S3Configuration.builder() .chunkedEncodingEnabled(false) .pathStyleAccessEnabled(express ? false : usePathStyle) .build()); + if (s3Properties.isS3ExpressImportReadEnabled() && !s3Properties.isDirectoryBucketEndpoint()) { + builder.disableS3ExpressSessionAuth(!express); + } if (!express && StringUtils.isNotBlank(endpointStr)) { if (!endpointStr.contains("://")) { @@ -218,18 +220,22 @@ private AwsCredentialsProvider getCredentialsProvider() { return credentialsProvider; } - private S3Client clientForHead(String requestBucket) throws IOException { - if (!requestBucket.endsWith(DIRECTORY_BUCKET_SUFFIX) + boolean usesS3ExpressRead(String requestBucket) throws IOException { + if (!s3Properties.isS3ExpressImportReadEnabled() + || requestBucket == null || !requestBucket.endsWith(DIRECTORY_BUCKET_SUFFIX) || StringUtils.isBlank(s3Properties.getEndpoint())) { - return getClient(); + return false; } Matcher endpoint = AWS_S3_REGIONAL_ENDPOINT_PATTERN.matcher(s3Properties.getEndpoint()); if (!endpoint.matches()) { + if (s3Properties.isDirectoryBucketEndpoint()) { + return false; + } if (isAwsEndpoint(s3Properties.getEndpoint())) { throw new IOException("AWS Directory Bucket requires a standard HTTPS regional S3 " + "endpoint; do not use s3express-control or a zonal endpoint"); } - return getClient(); + return false; } if (!isDirectoryBucketName(requestBucket)) { throw new IOException("Invalid AWS Directory Bucket name " + requestBucket @@ -242,6 +248,10 @@ private S3Client clientForHead(String requestBucket) throws IOException { if (usePathStyle) { throw new IOException("Path-style addressing is not supported for AWS Directory Bucket"); } + return true; + } + + private S3Client getExpressClient() throws IOException { if (closed.get()) { throw new IOException("S3ObjStorage is already closed"); } @@ -276,13 +286,20 @@ public RemoteObjects listObjects(String remotePath, String continuationToken) th @Override public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions options) throws IOException { ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); + boolean expressRead = usesS3ExpressRead(uri.bucket()); + String requestPrefix = expressRead ? slashTerminatedPrefix(uri.key()) : uri.key(); ListObjectsV2Request.Builder builder = ListObjectsV2Request.builder() - .bucket(uri.bucket()) - .prefix(uri.key()); + .bucket(uri.bucket()); + if (!expressRead || !requestPrefix.isEmpty()) { + builder.prefix(requestPrefix); + } if (options != null) { if (StringUtils.isNotBlank(options.continuationToken())) { builder.continuationToken(options.continuationToken()); } else if (StringUtils.isNotBlank(options.startAfter())) { + if (expressRead) { + throw new IOException("StartAfter is not supported for AWS Directory Bucket listings"); + } builder.startAfter(options.startAfter()); } if (options.maxKeys() > 0) { @@ -293,7 +310,8 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions } } try { - ListObjectsV2Response response = getClient().listObjectsV2(builder.build()); + S3Client listClient = expressRead ? getExpressClient() : getClient(); + ListObjectsV2Response response = listClient.listObjectsV2(builder.build()); List objects = response.contents().stream() .map(s3Obj -> new org.apache.doris.filesystem.spi.RemoteObject( s3Obj.key(), @@ -358,7 +376,7 @@ public RemoteObjects listObjectsNonRecursive(String remotePath, String continuat public org.apache.doris.filesystem.spi.RemoteObject headObject(String remotePath) throws IOException { ObjectStorageUri uri = ObjectStorageUri.parse(remotePath, usePathStyle, getSupportedSchemes()); try { - HeadObjectResponse response = clientForHead(uri.bucket()).headObject( + HeadObjectResponse response = getClient().headObject( HeadObjectRequest.builder().bucket(uri.bucket()).key(uri.key()).build()); return new org.apache.doris.filesystem.spi.RemoteObject( uri.key(), uri.key(), response.eTag(), response.contentLength(), @@ -726,6 +744,14 @@ private static boolean isDirectoryBucketName(String bucketName) { return bucketName != null && DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches(); } + private static String slashTerminatedPrefix(String key) { + if (key.isEmpty() || key.endsWith("/")) { + return key; + } + int slash = key.lastIndexOf('/'); + return slash < 0 ? "" : key.substring(0, slash + 1); + } + private static boolean isAwsEndpoint(String endpoint) { int schemeSeparator = endpoint.indexOf("://"); URI uri = URI.create(schemeSeparator > 0 ? endpoint : "https://" + endpoint); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java index fe9b28dd8c90b4..db24f228ff9177 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java @@ -45,6 +45,8 @@ */ class S3FileSystemTest { + private static final String DIRECTORY_BUCKET = "analytics--usw2-az1--x-s3"; + private S3ObjStorage mockStorage; private S3FileSystem fs; @@ -702,6 +704,96 @@ void globListWithLimit_directoryBucketFallsBackToSlashTerminatedStaticPrefix() t ArgumentMatchers.eq("s3://bucket/data/b"), ArgumentMatchers.any()); } + @Test + void globListWithLimit_regionalDirectoryBucketUsesSlashTerminatedPrefix() throws IOException { + S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( + "s3.endpoint", "https://s3.us-west-2.amazonaws.com", + "s3.region", "us-west-2")); + S3FileSystem directoryBucketFs = new S3FileSystem(properties, mockStorage); + Mockito.when(mockStorage.usesS3ExpressRead(DIRECTORY_BUCKET)).thenReturn(true); + Mockito.when(mockStorage.listObjects( + ArgumentMatchers.eq("s3://" + DIRECTORY_BUCKET + "/data/"), + ArgumentMatchers.isNull())) + .thenReturn(new RemoteObjects( + List.of( + new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L), + new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L)), + false, null)); + + GlobListing listing = directoryBucketFs.globListWithLimit( + Location.of("s3://" + DIRECTORY_BUCKET + "/data/[ab]*.csv"), null, 0L, 0L); + + Assertions.assertEquals(2, listing.getFiles().size()); + Assertions.assertEquals("data/", listing.getPrefix()); + Mockito.verify(mockStorage).listObjects( + ArgumentMatchers.eq("s3://" + DIRECTORY_BUCKET + "/data/"), + ArgumentMatchers.isNull()); + } + + @Test + void globListWithLimit_regionalDirectoryBucketExactPathFiltersSibling() throws IOException { + S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( + "s3.endpoint", "https://s3.us-west-2.amazonaws.com", + "s3.region", "us-west-2")); + S3FileSystem directoryBucketFs = new S3FileSystem(properties, mockStorage); + Mockito.when(mockStorage.usesS3ExpressRead(DIRECTORY_BUCKET)).thenReturn(true); + Mockito.when(mockStorage.listObjects( + ArgumentMatchers.eq("s3://" + DIRECTORY_BUCKET + "/data/"), + ArgumentMatchers.isNull())) + .thenReturn(new RemoteObjects( + List.of( + new RemoteObject("data/target.csv", "target.csv", null, 10L, 0L), + new RemoteObject("data/target.csv.bak", "target.csv.bak", null, 20L, 0L)), + false, null)); + + GlobListing listing = directoryBucketFs.globListWithLimit( + Location.of("s3://" + DIRECTORY_BUCKET + "/data/target.csv"), null, 0L, 0L); + + Assertions.assertEquals(1, listing.getFiles().size()); + Assertions.assertEquals( + "s3://" + DIRECTORY_BUCKET + "/data/target.csv", + listing.getFiles().get(0).location().uri()); + } + + @Test + void globListWithLimit_regionalDirectoryBucketPaginatesWithOpaqueToken() throws IOException { + S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( + "s3.endpoint", "https://s3.us-west-2.amazonaws.com", + "s3.region", "us-west-2")); + S3FileSystem directoryBucketFs = new S3FileSystem(properties, mockStorage); + String listUri = "s3://" + DIRECTORY_BUCKET + "/data/"; + Mockito.when(mockStorage.usesS3ExpressRead(DIRECTORY_BUCKET)).thenReturn(true); + Mockito.doAnswer(invocation -> { + ObjectListOptions options = invocation.getArgument(1); + if (options.continuationToken() == null) { + return new RemoteObjects( + List.of(new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L)), + true, "opaque-token"); + } + return new RemoteObjects( + List.of(new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L)), + false, null); + }).when(mockStorage).listObjectsWithOptions( + ArgumentMatchers.eq(listUri), ArgumentMatchers.any(ObjectListOptions.class)); + + GlobListing listing = directoryBucketFs.globListWithLimit( + Location.of("s3://" + DIRECTORY_BUCKET + "/data/*.csv"), null, 0L, 0L); + + Assertions.assertEquals(2, listing.getFiles().size()); + Assertions.assertEquals( + List.of( + "s3://" + DIRECTORY_BUCKET + "/data/b.csv", + "s3://" + DIRECTORY_BUCKET + "/data/a.csv"), + listing.getFiles().stream().map(file -> file.location().uri()).toList()); + ArgumentCaptor optionsCaptor = ArgumentCaptor.forClass(ObjectListOptions.class); + Mockito.verify(mockStorage, Mockito.times(2)).listObjectsWithOptions( + ArgumentMatchers.eq(listUri), optionsCaptor.capture()); + Assertions.assertNull(optionsCaptor.getAllValues().get(0).startAfter()); + Assertions.assertEquals( + "opaque-token", optionsCaptor.getAllValues().get(1).continuationToken()); + Assertions.assertNull(optionsCaptor.getAllValues().get(1).startAfter()); + } + @Test void globListWithLimit_listsExpandedDatePrefixesInsteadOfBroadDatePrefix() throws IOException { Mockito.when(mockStorage.listObjects(ArgumentMatchers.anyString(), ArgumentMatchers.isNull())) diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java index deefd3925a5855..ff664c368e442b 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java @@ -18,6 +18,7 @@ package org.apache.doris.filesystem.s3; import org.apache.doris.filesystem.UploadPartResult; +import org.apache.doris.filesystem.spi.ObjectListOptions; import org.apache.doris.filesystem.spi.RemoteObject; import org.apache.doris.filesystem.spi.RemoteObjects; import org.apache.doris.filesystem.spi.RequestBody; @@ -63,10 +64,13 @@ /** * Full unit tests for {@link S3ObjStorage} using a testable subclass that overrides - * {@link S3ObjStorage#buildClient()} to inject a mock S3Client. + * {@link S3ObjStorage#buildClient()} and {@link S3ObjStorage#buildExpressClient()} + * to inject mock S3 clients. */ class S3ObjStorageMockTest { + private static final String DIRECTORY_BUCKET = "analytics--usw2-az1--x-s3"; + private S3Client mockS3; private S3ObjStorage storage; @@ -150,6 +154,148 @@ void listObjects_truncatedResultReturnsContinuationToken() throws IOException { Assertions.assertEquals("next-token", result.getContinuationToken()); } + @Test + void listObjects_directoryBucketUsesExpressClientAndDirectoryPrefix() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + + expressStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/file-*.csv", null); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + Mockito.verify(expressClient).listObjectsV2(captor.capture()); + Assertions.assertEquals(DIRECTORY_BUCKET, captor.getValue().bucket()); + Assertions.assertEquals("data/", captor.getValue().prefix()); + Assertions.assertNull(captor.getValue().startAfter()); + Mockito.verify(regularClient, Mockito.never()).listObjectsV2( + ArgumentMatchers.any(ListObjectsV2Request.class)); + } + + @Test + void listObjects_directoryBucketUsesOpaqueContinuationToken() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + + expressStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/file.csv", "opaque-token"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + Mockito.verify(expressClient).listObjectsV2(captor.capture()); + Assertions.assertEquals("opaque-token", captor.getValue().continuationToken()); + Assertions.assertNull(captor.getValue().startAfter()); + } + + @Test + void listObjects_directoryBucketRootDoesNotSendPrefix() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + + expressStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/", null); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + Mockito.verify(expressClient).listObjectsV2(captor.capture()); + Assertions.assertNull(captor.getValue().prefix()); + Mockito.verifyNoInteractions(regularClient); + } + + @Test + void listObjects_directoryBucketWithoutImportMarkerStaysOnRegularClient() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage nonImportStorage = new TestableS3ObjStorage( + directoryBucketProperties("https://s3.us-west-2.amazonaws.com", "us-west-2", false), + regularClient, expressClient); + Mockito.when(regularClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + + nonImportStorage.listObjectsWithOptions( + "s3://" + DIRECTORY_BUCKET + "/data/file.csv", + ObjectListOptions.builder().startAfter("data/previous.csv").build()); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + Mockito.verify(regularClient).listObjectsV2(captor.capture()); + Assertions.assertEquals("data/file.csv", captor.getValue().prefix()); + Assertions.assertEquals("data/previous.csv", captor.getValue().startAfter()); + Mockito.verifyNoInteractions(expressClient); + } + + @Test + void listObjects_directoryBucketRejectsStartAfter() { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + + IOException exception = Assertions.assertThrows(IOException.class, + () -> expressStorage.listObjectsWithOptions( + "s3://" + DIRECTORY_BUCKET + "/data/file.csv", + ObjectListOptions.builder().startAfter("data/previous.csv").build())); + + Assertions.assertTrue(exception.getMessage().contains("StartAfter")); + Mockito.verifyNoInteractions(regularClient, expressClient); + } + + @Test + void listObjects_directorySuffixOnThirdPartyEndpointUsesRegularClient() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage compatibleStorage = directoryBucketStorage( + "https://minio.example.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(regularClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + + compatibleStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/file", null); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + Mockito.verify(regularClient).listObjectsV2(captor.capture()); + Assertions.assertEquals("data/file", captor.getValue().prefix()); + Mockito.verifyNoInteractions(expressClient); + } + + @Test + void usesS3ExpressRead_validatesNativeDirectoryBucketContext() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + + Assertions.assertTrue(directoryBucketStorage( + "s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + Assertions.assertFalse(directoryBucketStorage( + "https://minio.example.com", "us-west-2", false, + regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + Assertions.assertFalse(directoryBucketStorage( + "https://s3express-usw2-az1.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + + Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( + "http://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-east-1", false, + regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", true, + regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient).usesS3ExpressRead("invalid--x-s3")); + } + // ------------------------------------------------------------------ // headObject() // ------------------------------------------------------------------ @@ -171,6 +317,22 @@ void headObject_returnsRemoteObjectOnSuccess() throws IOException { Assertions.assertEquals("etag-xyz", result.getEtag()); } + @Test + void headObject_directoryBucketStaysOnRegularClient() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(regularClient.headObject(ArgumentMatchers.any(HeadObjectRequest.class))) + .thenReturn(HeadObjectResponse.builder().contentLength(1L).build()); + + expressStorage.headObject("s3://" + DIRECTORY_BUCKET + "/data/file.csv"); + + Mockito.verify(regularClient).headObject(ArgumentMatchers.any(HeadObjectRequest.class)); + Mockito.verifyNoInteractions(expressClient); + } + @Test void headObject_throwsFileNotFoundForNoSuchKeyException() { Mockito.when(mockS3.headObject(ArgumentMatchers.any(HeadObjectRequest.class))) @@ -209,6 +371,27 @@ void putObject_delegatesToS3Client() throws IOException { Assertions.assertEquals("obj", captor.getValue().key()); } + @Test + void putObject_directoryBucketStaysOnRegularClient() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(regularClient.putObject( + ArgumentMatchers.any(PutObjectRequest.class), + ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class))) + .thenReturn(PutObjectResponse.builder().build()); + + RequestBody body = RequestBody.of(new ByteArrayInputStream(new byte[]{1}), 1); + expressStorage.putObject("s3://" + DIRECTORY_BUCKET + "/data/file.csv", body); + + Mockito.verify(regularClient).putObject( + ArgumentMatchers.any(PutObjectRequest.class), + ArgumentMatchers.any(software.amazon.awssdk.core.sync.RequestBody.class)); + Mockito.verifyNoInteractions(expressClient); + } + // ------------------------------------------------------------------ // deleteObject() // ------------------------------------------------------------------ @@ -478,27 +661,77 @@ void close_closesS3Client() throws IOException { Mockito.verify(mockS3).close(); } + @Test + void close_closesRegularAndExpressClients() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "https://s3.us-west-2.amazonaws.com", "us-west-2", false, + regularClient, expressClient); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + expressStorage.getClient(); + expressStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/", null); + + expressStorage.close(); + + Mockito.verify(regularClient).close(); + Mockito.verify(expressClient).close(); + } + // ------------------------------------------------------------------ // Test infrastructure // ------------------------------------------------------------------ private static class TestableS3ObjStorage extends S3ObjStorage { private final S3Client mockClient; + private final S3Client mockExpressClient; TestableS3ObjStorage(Map properties, S3Client mockClient) { + this(properties, mockClient, mockClient); + } + + TestableS3ObjStorage(Map properties, S3Client mockClient, + S3Client mockExpressClient) { super(properties); this.mockClient = mockClient; + this.mockExpressClient = mockExpressClient; } TestableS3ObjStorage(S3FileSystemProperties properties, S3Client mockClient) { super(properties); this.mockClient = mockClient; + this.mockExpressClient = mockClient; } @Override protected S3Client buildClient() { return mockClient; } + + @Override + protected S3Client buildExpressClient() { + return mockExpressClient; + } + } + + private static S3ObjStorage directoryBucketStorage(String endpoint, String region, + boolean usePathStyle, S3Client regularClient, S3Client expressClient) { + Map props = directoryBucketProperties(endpoint, region, usePathStyle); + props.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); + return new TestableS3ObjStorage(props, regularClient, expressClient); + } + + private static Map directoryBucketProperties(String endpoint, String region, + boolean usePathStyle) { + Map props = new HashMap<>(); + props.put("AWS_ENDPOINT", endpoint); + props.put("AWS_REGION", region); + props.put("AWS_ACCESS_KEY", "testAK"); + props.put("AWS_SECRET_KEY", "testSK"); + props.put("AWS_BUCKET", DIRECTORY_BUCKET); + props.put("use_path_style", Boolean.toString(usePathStyle)); + return props; } private static class InspectableS3ObjStorage extends TestableS3ObjStorage { diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java index 5faee2bce6960d..773add5b7445e1 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/S3CompatibleFileSystem.java @@ -868,11 +868,12 @@ protected static List expandedGlobListPrefixes(String globPattern) { return prefixes == null ? List.of(longestNonGlobPrefix(globPattern)) : prefixes; } - protected String globListPrefix(String globPattern) { + protected String globListPrefix(String bucket, String globPattern) throws IOException { return longestNonGlobPrefix(globPattern); } - protected List globListPrefixes(String globPattern, String listPrefix) { + protected List globListPrefixes(String bucket, String globPattern, String listPrefix) + throws IOException { return expandedGlobListPrefixes(globPattern); } @@ -1205,8 +1206,8 @@ public GlobListing globListWithLimit(Location path, String startAfter, long maxB // separator is '\' which would corrupt object storage keys, and (b) Paths.get rejects keys // containing characters illegal in the host OS path syntax (':', '\', etc.). Pattern matcher = Pattern.compile(globToRegex(expandedKeyPattern)); - String listPrefix = globListPrefix(expandedKeyPattern); - List listPrefixes = globListPrefixes(expandedKeyPattern, listPrefix); + String listPrefix = globListPrefix(bucket, expandedKeyPattern); + List listPrefixes = globListPrefixes(bucket, expandedKeyPattern, listPrefix); List files = new ArrayList<>(); long totalSize = 0L; From ca40debac493b8bf8c062bd9694cbafb341cc327 Mon Sep 17 00:00:00 2001 From: Refrain Date: Sun, 19 Jul 2026 00:14:55 +0800 Subject: [PATCH 10/13] [refactor](s3) Narrow S3 Express support to imports ### What problem does this PR solve? Issue Number: None Related PR: #63409 Problem Summary: The previous S3 Express implementation inferred directory-bucket behavior from user endpoints and changed generic S3/import abstractions beyond the intended feature. This refactor limits S3 Express selection to trusted one-shot INSERT SELECT and Broker Load read paths with an explicit AWS provider and a complete directory bucket name. Doris ignores the user endpoint for these scoped reads, derives known Regions from the bucket Zone ID, falls back to an unchecked user Region for future Zone prefixes, and lets the AWS SDK resolve the zonal endpoint and manage S3 Express session credentials. Ordinary S3 and non-target import, catalog, resource, storage, and write paths retain their existing behavior. ### Release note Add S3 Express One Zone read support for one-shot INSERT SELECT from S3 and Broker Load with S3. Provide the complete directory bucket name and AWS provider; endpoint is ignored, while region is used only as an unchecked fallback for an unknown Zone prefix. ### Check List (For Author) - Test: Unit Test - BE S3ClientFactoryTest: 12 tests passed - FE targeted S3 Express and import tests: 196 tests passed - Behavior changed: Yes. Only the scoped S3 import reads select SDK-managed S3 Express endpoint and session authentication. - Does this need documentation: Yes. The PR description documents the supported entry points and configuration contract. --- be/src/io/fs/s3_file_system.cpp | 1 - be/src/util/s3_util.cpp | 384 ++++-------------- be/src/util/s3_util.h | 31 -- .../io/fs/s3_obj_stroage_client_mock_test.cpp | 29 +- be/test/io/s3_client_factory_test.cpp | 247 ++++++----- be/test/util/s3_util_test.cpp | 30 -- .../org/apache/doris/analysis/BrokerDesc.java | 14 + .../org/apache/doris/common/util/S3URI.java | 23 +- .../AbstractS3CompatibleProperties.java | 21 +- .../property/storage/S3Properties.java | 64 ++- .../apache/doris/fs/FileSystemFactory.java | 12 +- .../doris/load/loadv2/BrokerLoadJob.java | 18 +- .../nereids/parser/LogicalPlanBuilder.java | 18 +- .../trees/plans/commands/LoadCommand.java | 5 +- .../ExternalFileTableValuedFunction.java | 4 +- .../tablefunction/S3TableValuedFunction.java | 22 +- .../apache/doris/common/util/S3URITest.java | 1 + .../property/storage/S3PropertiesTest.java | 51 +-- .../FileSystemFactoryS3ExpressScopeTest.java | 75 ++++ .../doris/load/loadv2/BrokerLoadJobTest.java | 19 +- .../trees/plans/commands/LoadCommandTest.java | 30 ++ .../ExternalFileTableValuedFunctionTest.java | 9 +- .../doris/filesystem/s3/S3FileSystem.java | 8 +- .../filesystem/s3/S3FileSystemProperties.java | 37 +- .../filesystem/s3/S3FileSystemProvider.java | 4 + .../doris/filesystem/s3/S3ObjStorage.java | 167 ++++---- .../s3/S3FileSystemProviderTest.java | 22 + .../doris/filesystem/s3/S3FileSystemTest.java | 27 -- .../filesystem/s3/S3ObjStorageMockTest.java | 214 ++++++++-- .../doris/filesystem/s3/S3ObjStorageTest.java | 26 ++ 30 files changed, 833 insertions(+), 780 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 88ef46e61d6e18..63be8f1955a59c 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -101,7 +101,6 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.max_connections = conf.max_connections; reset_conf.request_timeout_ms = conf.request_timeout_ms; reset_conf.use_virtual_addressing = conf.use_virtual_addressing; - reset_conf.enable_s3_express_read = conf.enable_s3_express_read; reset_conf.role_arn = conf.role_arn; reset_conf.external_id = conf.external_id; diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index a22bf69ee9f8d7..046fb3d4bfe0cf 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -22,7 +22,6 @@ #include #include #include -#include #include #include #include @@ -30,16 +29,13 @@ #include #include #include -#ifdef BE_TEST -#include -#endif #include #include #include -#include #include -#include +#include +#include #include "util/string_util.h" @@ -89,215 +85,92 @@ bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); namespace { -std::string ascii_lower(std::string_view value) { - std::string result(value); - std::ranges::transform(result, result.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - return result; -} - -struct AwsS3Endpoint { - bool aws_domain = false; - bool https = false; - bool standard_regional = false; - std::string region; -}; - -AwsS3Endpoint parse_aws_s3_endpoint(std::string_view endpoint) { - AwsS3Endpoint result; - const auto scheme_separator = endpoint.find("://"); - if (scheme_separator != std::string_view::npos) { - result.https = ascii_lower(endpoint.substr(0, scheme_separator)) == "https"; - endpoint.remove_prefix(scheme_separator + 3); - } else { - result.https = true; +std::optional get_s3_express_zone_id(std::string_view bucket) { + constexpr std::string_view suffix = "--x-s3"; + if (!bucket.ends_with(suffix)) { + return std::nullopt; } - - const auto path_separator = endpoint.find_first_of("/?#"); - const auto authority = endpoint.substr(0, path_separator); - const auto path = path_separator == std::string_view::npos ? std::string_view {} - : endpoint.substr(path_separator); - bool standard_port = true; - auto host = authority; - if (const auto port_separator = authority.rfind(':'); - port_separator != std::string_view::npos) { - standard_port = authority.substr(port_separator + 1) == "443"; - host = authority.substr(0, port_separator); + bucket.remove_suffix(suffix.size()); + const auto zone_separator = bucket.rfind("--"); + if (zone_separator == std::string_view::npos || zone_separator == 0 || + zone_separator + 2 == bucket.size()) { + return std::nullopt; } - - const auto lower_host = ascii_lower(host); - constexpr std::string_view aws_domain = ".amazonaws.com"; - constexpr std::string_view aws_china_domain = ".amazonaws.com.cn"; - constexpr std::string_view aws_dualstack_domain = ".api.aws"; - std::string_view domain; - if (lower_host.ends_with(aws_china_domain)) { - domain = aws_china_domain; - } else if (lower_host.ends_with(aws_domain)) { - domain = aws_domain; - } else if (lower_host.ends_with(aws_dualstack_domain)) { - result.aws_domain = true; - return result; - } else { - return result; + const auto zone_id = bucket.substr(zone_separator + 2); + const auto az_separator = zone_id.rfind("-az"); + if (az_separator == std::string_view::npos || az_separator == 0 || + az_separator + 3 == zone_id.size()) { + return std::nullopt; } - result.aws_domain = true; - - constexpr std::string_view regional_prefix = "s3."; - if (!result.https || !standard_port || (!path.empty() && path != "/") || - !lower_host.starts_with(regional_prefix)) { - return result; + for (const char c : zone_id.substr(0, az_separator)) { + if (!((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-')) { + return std::nullopt; + } } - const auto region = std::string_view(lower_host) - .substr(regional_prefix.size(), - lower_host.size() - regional_prefix.size() - domain.size()); - if (region.empty() || region.find('.') != std::string_view::npos) { - return result; + for (const char c : zone_id.substr(az_separator + 3)) { + if (c < '0' || c > '9') { + return std::nullopt; + } } - result.standard_regional = true; - result.region = region; - return result; -} - -bool has_s3_directory_bucket_suffix(std::string_view bucket) { - return bucket.ends_with("--x-s3"); + return zone_id; } -class S3CompatibleEndpointProvider final : public Aws::S3::S3EndpointProvider { -public: - explicit S3CompatibleEndpointProvider(std::string_view endpoint) { - std::string normalized_endpoint(endpoint); - if (normalized_endpoint.find("://") == std::string::npos) { - normalized_endpoint.insert(0, "https://"); +std::optional infer_s3_express_region(std::string_view zone_id) { + // Unknown zone prefixes deliberately fall back to the user-provided region below, so newly + // introduced AWS regions do not require Doris-side validation before the SDK can use them. + static constexpr std::pair region_mappings[] = { + {"use1", "us-east-1"}, {"use2", "us-east-2"}, {"usw2", "us-west-2"}, + {"aps1", "ap-south-1"}, {"apne1", "ap-northeast-1"}, {"euw1", "eu-west-1"}, + {"eun1", "eu-north-1"}, + }; + const auto separator = zone_id.find('-'); + const auto zone_prefix = zone_id.substr(0, separator); + for (const auto& [prefix, region] : region_mappings) { + if (zone_prefix == prefix) { + return region; } - _endpoint_authority = Aws::Http::URI(normalized_endpoint.c_str()).GetAuthority(); - DORIS_CHECK(!_endpoint_authority.empty()); } + return std::nullopt; +} - Aws::Endpoint::ResolveEndpointOutcome ResolveEndpoint( - const Aws::Endpoint::EndpointParameters& endpoint_parameters) const override { - auto compatible_parameters = endpoint_parameters; - Aws::String bucket; - Aws::String surrogate_bucket; - for (auto& parameter : compatible_parameters) { - Aws::String value; - if (parameter.GetName() == "Bucket" && - parameter.GetString(value) == - Aws::Endpoint::EndpointParameter::GetSetResult::SUCCESS && - has_s3_directory_bucket_suffix(std::string_view(value.data(), value.size()))) { - bucket = std::move(value); - surrogate_bucket = bucket; - // The SDK selects S3 Express endpoint/signing rules solely from this suffix. - // Resolve an otherwise identical bucket through regular S3 rules, then restore - // the real bucket in the resolved URI below. - surrogate_bucket.back() = '2'; - parameter.SetString(surrogate_bucket); - break; - } - } - - auto outcome = Aws::S3::S3EndpointProvider::ResolveEndpoint(compatible_parameters); - if (bucket.empty() || !outcome.IsSuccess()) { - return outcome; - } +bool use_s3_express_client(const S3ClientConf& conf) { + return conf.enable_s3_express_read && conf.provider == io::ObjStorageType::AWS && + get_s3_express_zone_id(conf.bucket).has_value(); +} - auto& endpoint = outcome.GetResult(); - auto uri = endpoint.GetURI(); - auto authority = uri.GetAuthority(); - auto path = uri.GetPath(); - const auto bucket_pos = path.rfind(surrogate_bucket); - const bool bucket_is_path_segment = bucket_pos != Aws::String::npos && - (bucket_pos == 0 || path[bucket_pos - 1] == '/') && - (bucket_pos + surrogate_bucket.size() == path.size() || - path[bucket_pos + surrogate_bucket.size()] == '/'); - const auto virtual_host_authority = surrogate_bucket + "." + _endpoint_authority; - if (authority == _endpoint_authority) { - DORIS_CHECK(bucket_is_path_segment); - path.replace(bucket_pos, surrogate_bucket.size(), bucket); - uri.SetPath(path); - } else { - DORIS_CHECK(authority == virtual_host_authority); - authority.replace(0, surrogate_bucket.size(), bucket); - uri.SetAuthority(authority); +std::string_view effective_s3_region(const S3ClientConf& conf) { + if (use_s3_express_client(conf)) { + if (const auto zone_id = get_s3_express_zone_id(conf.bucket); zone_id.has_value()) { + if (const auto region = infer_s3_express_region(*zone_id); region.has_value()) { + return *region; + } } - endpoint.SetURI(std::move(uri)); - return outcome; } - -private: - Aws::String _endpoint_authority; -}; - -bool is_legacy_s3_directory_bucket_endpoint(std::string_view endpoint) { - const auto lower_endpoint = ascii_lower(endpoint); - return lower_endpoint.find("s3express-control.") != std::string::npos || - lower_endpoint.find("s3express-") != std::string::npos; + return conf.region; } -bool is_s3_directory_bucket_name(std::string_view bucket) { - constexpr std::string_view suffix = "--x-s3"; - if (!bucket.ends_with(suffix)) { - return false; - } - bucket.remove_suffix(suffix.size()); - const auto zone_separator = bucket.rfind("--"); - if (zone_separator == std::string_view::npos || zone_separator == 0) { - return false; - } - const auto base_name = bucket.substr(0, zone_separator); - if (!std::isalnum(static_cast(base_name.front())) || - !std::isalnum(static_cast(base_name.back())) || - !std::ranges::all_of(base_name, [](unsigned char c) { - return std::islower(c) || std::isdigit(c) || c == '-'; - })) { - return false; +void configure_s3_express_import_read(const StringCaseMap& properties, + S3ClientConf* conf) { + const auto express_import = properties.find(S3_EXPRESS_IMPORT_READ); + const auto express_zone_id = get_s3_express_zone_id(conf->bucket); + conf->enable_s3_express_read = + express_import != properties.end() && express_import->second == "true" && + conf->provider == io::ObjStorageType::AWS && express_zone_id.has_value(); + if (conf->enable_s3_express_read) { + if (const auto region = infer_s3_express_region(*express_zone_id); region.has_value()) { + conf->region = std::string(*region); + } } - const auto zone_id = bucket.substr(zone_separator + 2); - const auto az_separator = zone_id.rfind("-az"); - return az_separator != std::string_view::npos && az_separator > 0 && - az_separator + 3 < zone_id.size() && - std::ranges::all_of(zone_id.substr(0, az_separator), - [](unsigned char c) { - return std::islower(c) || std::isdigit(c) || c == '-'; - }) && - std::ranges::all_of(zone_id.substr(az_separator + 3), - [](unsigned char c) { return std::isdigit(c); }); } doris::Status is_s3_conf_valid(const S3ClientConf& conf) { - if (conf.endpoint.empty()) { + const bool s3_express = use_s3_express_client(conf); + if (conf.endpoint.empty() && !s3_express) { return Status::InvalidArgument("Invalid s3 conf, empty endpoint"); } - if (conf.region.empty()) { + if (effective_s3_region(conf).empty()) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } - const auto endpoint = parse_aws_s3_endpoint(conf.endpoint); - if (conf.enable_s3_express_read && conf.provider == io::ObjStorageType::AWS && - endpoint.aws_domain && has_s3_directory_bucket_suffix(conf.bucket) && - !is_legacy_s3_directory_bucket_endpoint(conf.endpoint)) { - if (!is_s3_directory_bucket_name(conf.bucket)) { - return Status::InvalidArgument( - "Invalid AWS Directory Bucket name {}; expected " - "----x-s3", - conf.bucket); - } - if (!endpoint.https) { - return Status::InvalidArgument("AWS Directory Bucket requires HTTPS"); - } - if (!endpoint.standard_regional) { - return Status::InvalidArgument( - "AWS Directory Bucket requires a standard regional S3 endpoint; do not use " - "s3express-control or a zonal endpoint"); - } - if (endpoint.region != ascii_lower(conf.region)) { - return Status::InvalidArgument( - "AWS Directory Bucket endpoint region {} does not match configured region {}", - endpoint.region, conf.region); - } - if (!conf.use_virtual_addressing) { - return Status::InvalidArgument( - "Path-style addressing is not supported for AWS Directory Bucket"); - } - } if (conf.role_arn.empty()) { // Allow anonymous access when both ak and sk are empty @@ -696,15 +569,18 @@ std::shared_ptr S3ClientFactory::get_aws_cred std::shared_ptr S3ClientFactory::_create_s3_client( const S3ClientConf& s3_conf) { - const auto build_options = get_s3_client_build_options(s3_conf); + const bool s3_express = use_s3_express_client(s3_conf); TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", std::make_shared(std::make_shared())); Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); - if (build_options.override_endpoint) { + // Let the SDK derive the zonal endpoint from the complete directory bucket name. A user + // endpoint is intentionally ignored for this scoped Express import client. + if (!s3_express && s3_conf.need_override_endpoint) { aws_config.endpointOverride = s3_conf.endpoint; } - aws_config.region = s3_conf.region; + const auto region = effective_s3_region(s3_conf); + aws_config.region = Aws::String(region.data(), region.size()); if (_ca_cert_file_path.empty()) { _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); @@ -729,7 +605,7 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms; } - if (build_options.force_https) { + if (s3_express) { aws_config.scheme = Aws::Http::Scheme::HTTPS; } else if (config::s3_client_http_scheme == "http") { aws_config.scheme = Aws::Http::Scheme::HTTP; @@ -739,24 +615,16 @@ std::shared_ptr S3ClientFactory::_create_s3_client( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true); std::shared_ptr new_client; - if (build_options.use_endpoint_provider) { - const auto signing_policy = - build_options.request_dependent_signing - ? Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent - : Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; - Aws::S3::S3ClientConfiguration endpoint_provider_config( - aws_config, signing_policy, build_options.use_virtual_addressing); - endpoint_provider_config.disableS3ExpressAuth = build_options.disable_s3_express_auth; - std::shared_ptr endpoint_provider; - if (build_options.use_compatible_endpoint_provider) { - endpoint_provider = - Aws::MakeShared("S3Client", s3_conf.endpoint); - } else { - endpoint_provider = Aws::MakeShared("S3Client"); - } - new_client = std::make_shared(get_aws_credentials_provider(s3_conf), - std::move(endpoint_provider), - endpoint_provider_config); + if (s3_express) { + // Directory buckets use virtual-hosted endpoints. Do not propagate a user path-style + // preference into the SDK client selected by the directory bucket name. + Aws::S3::S3ClientConfiguration express_config( + aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + true /* use virtual addressing */); + express_config.disableS3ExpressAuth = false; + new_client = std::make_shared( + get_aws_credentials_provider(s3_conf), + Aws::MakeShared("S3Client"), express_config); } else { new_client = std::make_shared( get_aws_credentials_provider(s3_conf), std::move(aws_config), @@ -821,10 +689,7 @@ Status S3ClientFactory::convert_properties_to_s3_conf( s3_conf->bucket = s3_uri.get_bucket(); // For azure's compatibility s3_conf->client_conf.bucket = s3_uri.get_bucket(); - const auto express_import = properties.find(S3_EXPRESS_IMPORT_READ); - s3_conf->client_conf.enable_s3_express_read = express_import != properties.end() && - express_import->second == "true" && - has_s3_directory_bucket_suffix(s3_conf->bucket); + configure_s3_express_import_read(properties, &s3_conf->client_conf); s3_conf->prefix = ""; // See https://sdk.amazonaws.com/cpp/api/LATEST/class_aws_1_1_s3_1_1_s3_client.html @@ -846,6 +711,7 @@ Status S3ClientFactory::convert_properties_to_s3_conf( if (auto it = properties.find(S3_CREDENTIALS_PROVIDER_TYPE); it != properties.end()) { s3_conf->client_conf.cred_provider_type = cred_provider_type_from_string(it->second); } + if (auto st = is_s3_conf_valid(s3_conf->client_conf); !st.ok()) { return st; } @@ -1029,86 +895,4 @@ std::string hide_access_key(const std::string& ak) { return key; } -bool is_s3_express(std::string_view bucket, std::string_view endpoint, std::string_view region) { - const auto parsed_endpoint = parse_aws_s3_endpoint(endpoint); - return is_s3_directory_bucket_name(bucket) && parsed_endpoint.standard_regional && - parsed_endpoint.region == ascii_lower(region); -} - -S3ClientBuildOptions get_s3_client_build_options(const S3ClientConf& conf) { - const bool s3_express = conf.enable_s3_express_read && - conf.provider == io::ObjStorageType::AWS && - is_s3_express(conf.bucket, conf.endpoint, conf.region); - if (s3_express) { - return {.use_endpoint_provider = true, - .use_compatible_endpoint_provider = false, - .override_endpoint = false, - .force_https = true, - .use_virtual_addressing = true, - .request_dependent_signing = true, - .disable_s3_express_auth = false}; - } - - const bool compatible_directory_bucket = conf.enable_s3_express_read && - conf.need_override_endpoint && - has_s3_directory_bucket_suffix(conf.bucket) && - !parse_aws_s3_endpoint(conf.endpoint).aws_domain; - if (compatible_directory_bucket) { - return {.use_endpoint_provider = true, - .use_compatible_endpoint_provider = true, - .override_endpoint = conf.need_override_endpoint, - .force_https = false, - .use_virtual_addressing = conf.use_virtual_addressing, - .request_dependent_signing = false, - .disable_s3_express_auth = true}; - } - - return {.use_endpoint_provider = false, - .use_compatible_endpoint_provider = false, - .override_endpoint = conf.need_override_endpoint, - .force_https = false, - .use_virtual_addressing = conf.use_virtual_addressing, - .request_dependent_signing = false, - .disable_s3_express_auth = false}; -} - -#ifdef BE_TEST -S3EndpointResolutionForTest resolve_s3_compatible_endpoint_for_test(std::string_view endpoint, - std::string_view region, - std::string_view bucket, - bool use_virtual_addressing) { - Aws::Client::ClientConfiguration client_config; - client_config.endpointOverride = std::string(endpoint); - client_config.region = std::string(region); - Aws::S3::S3ClientConfiguration s3_config( - client_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - use_virtual_addressing); - s3_config.disableS3ExpressAuth = true; - - S3CompatibleEndpointProvider provider(endpoint); - provider.InitBuiltInParameters(s3_config); - provider.AccessClientContextParameters().SetForcePathStyle(!use_virtual_addressing); - provider.AccessClientContextParameters().SetDisableS3ExpressSessionAuth(true); - provider.OverrideEndpoint(client_config.endpointOverride); - - Aws::S3::Model::GetObjectRequest request; - request.SetBucket(std::string(bucket)); - request.SetKey("object"); - auto outcome = provider.ResolveEndpoint(request.GetEndpointContextParams()); - DORIS_CHECK(outcome.IsSuccess()); - const auto& endpoint_result = outcome.GetResult(); - const auto& attributes = endpoint_result.GetAttributes(); - DORIS_CHECK(attributes.has_value()); - const auto& signing_name = attributes->authScheme.GetSigningName(); - DORIS_CHECK(signing_name.has_value()); - return {.url = endpoint_result.GetURL(), - .authority = endpoint_result.GetURI().GetAuthority(), - .path = endpoint_result.GetURI().GetPath(), - .backend = attributes->backend, - .signing_name = signing_name.value(), - .port = endpoint_result.GetURI().GetPort(), - .use_s3_express_auth = attributes->useS3ExpressAuth}; -} -#endif - } // end namespace doris diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index 1bcc10129ebcbc..a4c1b8498fa0f8 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -30,7 +30,6 @@ #include #include #include -#include #include #include "common/status.h" @@ -66,7 +65,6 @@ extern bvar::LatencyRecorder s3_copy_object_latency; }; // namespace s3_bvar std::string hide_access_key(const std::string& ak); -bool is_s3_express(std::string_view bucket, std::string_view endpoint, std::string_view region); int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_burst, size_t limit); int64_t apply_s3_rate_limit(S3RateLimitType type); // Rebuild the S3 GET/PUT rate limiters if the related configs have changed. @@ -128,35 +126,6 @@ struct S3ClientConf { } }; -struct S3ClientBuildOptions { - bool use_endpoint_provider = false; - bool use_compatible_endpoint_provider = false; - bool override_endpoint = false; - bool force_https = false; - bool use_virtual_addressing = true; - bool request_dependent_signing = false; - bool disable_s3_express_auth = false; -}; - -S3ClientBuildOptions get_s3_client_build_options(const S3ClientConf& conf); - -#ifdef BE_TEST -struct S3EndpointResolutionForTest { - std::string url; - std::string authority; - std::string path; - std::string backend; - std::string signing_name; - uint16_t port = 0; - bool use_s3_express_auth = false; -}; - -S3EndpointResolutionForTest resolve_s3_compatible_endpoint_for_test(std::string_view endpoint, - std::string_view region, - std::string_view bucket, - bool use_virtual_addressing); -#endif - struct S3Conf { std::string bucket; std::string prefix; diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index a23565af378e0f..b7e635c1f1d29f 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -17,8 +17,6 @@ #include #include -#include -#include #include #include #include @@ -37,8 +35,6 @@ class MockS3Client : public Aws::S3::S3Client { MOCK_METHOD(Aws::S3::Model::ListObjectsV2Outcome, ListObjectsV2, (const Aws::S3::Model::ListObjectsV2Request& request), (const, override)); - MOCK_METHOD(Aws::S3::Model::GetObjectOutcome, GetObject, - (const Aws::S3::Model::GetObjectRequest& request), (const, override)); }; class S3ObjStorageClientMockTest : public testing::Test { @@ -124,32 +120,9 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { files.clear(); } -TEST_F(S3ObjStorageClientMockTest, get_object_sets_range) { - auto mock_s3_client = std::make_shared(); - S3ObjStorageClient s3_obj_storage_client(mock_s3_client); - - EXPECT_CALL(*mock_s3_client, GetObject(testing::_)) - .WillOnce([](const GetObjectRequest& request) { - EXPECT_EQ(request.GetBucket(), "dummy-bucket"); - EXPECT_EQ(request.GetKey(), "data.parquet"); - EXPECT_EQ(request.GetRange(), "bytes=7-10"); - GetObjectResult result; - result.SetContentLength(4); - return GetObjectOutcome(std::move(result)); - }); - - char buffer[4]; - size_t size_return = 0; - auto response = - s3_obj_storage_client.get_object({.bucket = "dummy-bucket", .key = "data.parquet"}, - buffer, 7, sizeof(buffer), &size_return); - EXPECT_EQ(ErrorCode::OK, response.status.code); - EXPECT_EQ(sizeof(buffer), size_return); -} - TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { auto path = doris::get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); LOG(INFO) << "config:" << config::ca_cert_file_paths << " path:" << path; ASSERT_FALSE(path.empty()); } -} // namespace doris::io +} // namespace doris::io \ No newline at end of file diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 68c26e58df4e6e..f99c4a37841da8 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -18,6 +18,9 @@ #include #include #include +#include +#include +#include #include #include @@ -254,41 +257,100 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ExpressReadConf) { S3Conf s3_conf; ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); ASSERT_TRUE(s3_conf.client_conf.enable_s3_express_read); + ASSERT_EQ(s3_conf.client_conf.region, "us-west-2"); - properties["AWS_ENDPOINT"] = "s3.us-west-2.amazonaws.com"; - ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); - - properties["AWS_ENDPOINT"] = "http://s3.us-west-2.amazonaws.com"; - ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); - - properties["AWS_ENDPOINT"] = "https://s3express-control.us-west-2.amazonaws.com"; - ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); - - properties["AWS_ENDPOINT"] = - "https://analytics--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"; - ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); - - properties["AWS_ENDPOINT"] = "https://s3.us-west-2.amazonaws.com"; + properties["AWS_ENDPOINT"] = "http://endpoint-is-ignored.example.com"; properties["AWS_REGION"] = "us-east-1"; - ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); - - properties["AWS_REGION"] = "us-west-2"; properties["use_path_style"] = "true"; - ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + S3Conf ignored_properties_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, + &ignored_properties_conf) + .ok()); + EXPECT_EQ(ignored_properties_conf.client_conf.endpoint, + "http://endpoint-is-ignored.example.com"); + EXPECT_EQ(ignored_properties_conf.client_conf.region, "us-west-2"); + EXPECT_FALSE(ignored_properties_conf.client_conf.use_virtual_addressing); - properties["AWS_ENDPOINT"] = "https://minio.example.com"; - ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); - ASSERT_TRUE(s3_conf.client_conf.enable_s3_express_read); - ASSERT_FALSE(is_s3_express(s3_conf.client_conf.bucket, s3_conf.client_conf.endpoint, - s3_conf.client_conf.region)); + properties.clear(); + properties[S3_EXPRESS_IMPORT_READ] = "true"; + S3Conf bucket_only_conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &bucket_only_conf) + .ok()); + EXPECT_TRUE(bucket_only_conf.client_conf.endpoint.empty()); + EXPECT_EQ(bucket_only_conf.client_conf.region, "us-west-2"); + + S3URI future_region_uri("s3://analytics--future1-az1--x-s3/path/to/data.parquet"); + ASSERT_TRUE(future_region_uri.parse().ok()); + properties["AWS_REGION"] = "future-region-1"; + S3Conf future_region_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, future_region_uri, + &future_region_conf) + .ok()); + EXPECT_TRUE(future_region_conf.client_conf.enable_s3_express_read); + EXPECT_EQ(future_region_conf.client_conf.region, "future-region-1"); + + properties.erase("AWS_REGION"); + S3Conf missing_region_conf; + ASSERT_FALSE(S3ClientFactory::convert_properties_to_s3_conf(properties, future_region_uri, + &missing_region_conf) + .ok()); S3URI ordinary_uri("s3://ordinary-bucket/path/to/data.parquet"); ASSERT_TRUE(ordinary_uri.parse().ok()); properties["AWS_ENDPOINT"] = "https://s3.us-west-2.amazonaws.com"; - properties.erase("use_path_style"); - ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, ordinary_uri, &s3_conf) + properties["AWS_REGION"] = "us-west-2"; + S3Conf ordinary_conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties, ordinary_uri, &ordinary_conf) + .ok()); + ASSERT_FALSE(ordinary_conf.client_conf.enable_s3_express_read); + + S3URI malformed_uri("s3://analytics--x-s3/path/to/data.parquet"); + ASSERT_TRUE(malformed_uri.parse().ok()); + S3Conf malformed_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, malformed_uri, + &malformed_conf) + .ok()); + ASSERT_FALSE(malformed_conf.client_conf.enable_s3_express_read); + + S3URI malformed_zone_uri("s3://analytics--usw2-azx--x-s3/path/to/data.parquet"); + ASSERT_TRUE(malformed_zone_uri.parse().ok()); + S3Conf malformed_zone_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, malformed_zone_uri, + &malformed_zone_conf) .ok()); - ASSERT_FALSE(s3_conf.client_conf.enable_s3_express_read); + ASSERT_FALSE(malformed_zone_conf.client_conf.enable_s3_express_read); + + properties["provider"] = "AZURE"; + S3Conf non_aws_conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &non_aws_conf).ok()); + ASSERT_FALSE(non_aws_conf.client_conf.enable_s3_express_read); +} + +TEST_F(S3ClientFactoryTest, S3ExpressRegionInference) { + struct TestCase { + const char* zone_id; + const char* region; + }; + const TestCase test_cases[] = { + {.zone_id = "use1-az4", .region = "us-east-1"}, + {.zone_id = "use2-az1", .region = "us-east-2"}, + {.zone_id = "usw2-az1", .region = "us-west-2"}, + {.zone_id = "aps1-az1", .region = "ap-south-1"}, + {.zone_id = "apne1-az1", .region = "ap-northeast-1"}, + {.zone_id = "euw1-az1", .region = "eu-west-1"}, + {.zone_id = "eun1-az1", .region = "eu-north-1"}, + }; + const std::map properties {{S3_EXPRESS_IMPORT_READ, "true"}}; + for (const auto& test_case : test_cases) { + S3URI uri(fmt::format("s3://analytics--{}--x-s3/data.parquet", test_case.zone_id)); + ASSERT_TRUE(uri.parse().ok()); + S3Conf conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, uri, &conf).ok()); + EXPECT_EQ(conf.client_conf.region, test_case.region); + } } TEST_F(S3ClientFactoryTest, S3ExpressReadIsPartOfClientCacheKey) { @@ -303,109 +365,36 @@ TEST_F(S3ClientFactoryTest, S3ExpressReadIsPartOfClientCacheKey) { ASSERT_NE(regular_conf.get_hash(), express_read_conf.get_hash()); } -TEST_F(S3ClientFactoryTest, S3ExpressReadClientBuildOptions) { - S3ClientConf conf; - conf.bucket = "analytics--usw2-az1--x-s3"; - conf.endpoint = "https://s3.us-west-2.amazonaws.com"; - conf.region = "us-west-2"; - conf.enable_s3_express_read = true; - - auto options = get_s3_client_build_options(conf); - EXPECT_TRUE(options.use_endpoint_provider); - EXPECT_FALSE(options.use_compatible_endpoint_provider); - EXPECT_FALSE(options.override_endpoint); - EXPECT_TRUE(options.force_https); - EXPECT_TRUE(options.use_virtual_addressing); - EXPECT_TRUE(options.request_dependent_signing); - EXPECT_FALSE(options.disable_s3_express_auth); - - conf.endpoint = "https://minio.example.com"; - conf.use_virtual_addressing = false; - options = get_s3_client_build_options(conf); - EXPECT_TRUE(options.use_endpoint_provider); - EXPECT_TRUE(options.use_compatible_endpoint_provider); - EXPECT_TRUE(options.override_endpoint); - EXPECT_FALSE(options.force_https); - EXPECT_FALSE(options.use_virtual_addressing); - EXPECT_FALSE(options.request_dependent_signing); - EXPECT_TRUE(options.disable_s3_express_auth); - - conf.need_override_endpoint = false; - options = get_s3_client_build_options(conf); - EXPECT_FALSE(options.use_endpoint_provider); - EXPECT_FALSE(options.use_compatible_endpoint_provider); - EXPECT_FALSE(options.override_endpoint); - EXPECT_FALSE(options.disable_s3_express_auth); - conf.need_override_endpoint = true; - - conf.endpoint = "https://s3express-control.us-west-2.amazonaws.com"; - conf.use_virtual_addressing = true; - options = get_s3_client_build_options(conf); - EXPECT_FALSE(options.use_endpoint_provider); - EXPECT_FALSE(options.use_compatible_endpoint_provider); - EXPECT_TRUE(options.override_endpoint); - EXPECT_FALSE(options.force_https); - EXPECT_TRUE(options.use_virtual_addressing); - EXPECT_FALSE(options.request_dependent_signing); - EXPECT_FALSE(options.disable_s3_express_auth); - - conf.bucket = "ordinary-bucket"; - conf.endpoint = "https://s3.us-west-2.amazonaws.com"; - options = get_s3_client_build_options(conf); - EXPECT_FALSE(options.use_endpoint_provider); - EXPECT_FALSE(options.use_compatible_endpoint_provider); - EXPECT_TRUE(options.override_endpoint); - EXPECT_FALSE(options.force_https); - EXPECT_TRUE(options.use_virtual_addressing); - EXPECT_FALSE(options.request_dependent_signing); - EXPECT_FALSE(options.disable_s3_express_auth); -} - -TEST_F(S3ClientFactoryTest, S3CompatibleDirectoryBucketUsesRegularEndpointRules) { - constexpr std::string_view bucket = "analytics--usw2-az1--x-s3"; - - auto virtual_host = resolve_s3_compatible_endpoint_for_test("https://minio.example.com", - "us-west-2", bucket, true); - EXPECT_EQ(virtual_host.authority, fmt::format("{}.minio.example.com", bucket)); - EXPECT_EQ(virtual_host.signing_name, "s3"); - EXPECT_TRUE(virtual_host.backend.empty()); - EXPECT_FALSE(virtual_host.use_s3_express_auth); - EXPECT_EQ(virtual_host.url.find("--x-s2"), std::string::npos); - - auto path_style = resolve_s3_compatible_endpoint_for_test("https://minio.example.com/base", - "us-west-2", bucket, false); - EXPECT_EQ(path_style.authority, "minio.example.com"); - EXPECT_EQ(path_style.path, fmt::format("/base/{}", bucket)); - EXPECT_EQ(path_style.signing_name, "s3"); - EXPECT_TRUE(path_style.backend.empty()); - EXPECT_FALSE(path_style.use_s3_express_auth); - - auto ip_endpoint = resolve_s3_compatible_endpoint_for_test("http://127.0.0.1:9000", "us-west-2", - bucket, true); - EXPECT_EQ(ip_endpoint.authority, "127.0.0.1"); - EXPECT_EQ(ip_endpoint.port, 9000); - EXPECT_TRUE(ip_endpoint.path.ends_with(fmt::format("/{}", bucket))); - EXPECT_EQ(ip_endpoint.signing_name, "s3"); - EXPECT_TRUE(ip_endpoint.backend.empty()); - EXPECT_FALSE(ip_endpoint.use_s3_express_auth); - - constexpr std::string_view surrogate_bucket = "analytics--usw2-az1--x-s2"; - auto colliding_path_style = resolve_s3_compatible_endpoint_for_test( - fmt::format("https://{}.example.com/base", surrogate_bucket), "us-west-2", bucket, - false); - EXPECT_EQ(colliding_path_style.authority, fmt::format("{}.example.com", surrogate_bucket)); - EXPECT_EQ(colliding_path_style.path, fmt::format("/base/{}", bucket)); - - constexpr std::string_view dotted_bucket = "a.b--x-s3"; - constexpr std::string_view dotted_surrogate_bucket = "a.b--x-s2"; - auto auto_path_style = resolve_s3_compatible_endpoint_for_test( - fmt::format("https://{}.example.com/base", dotted_surrogate_bucket), "us-west-2", - dotted_bucket, true); - EXPECT_EQ(auto_path_style.authority, fmt::format("{}.example.com", dotted_surrogate_bucket)); - EXPECT_EQ(auto_path_style.path, fmt::format("/base/{}", dotted_bucket)); - EXPECT_EQ(auto_path_style.signing_name, "s3"); - EXPECT_TRUE(auto_path_style.backend.empty()); - EXPECT_FALSE(auto_path_style.use_s3_express_auth); +TEST_F(S3ClientFactoryTest, S3ExpressUsesSdkEndpointAndSessionAuth) { + Aws::S3::S3ClientConfiguration config(S3ClientFactory::getClientConfiguration(), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + true); + config.region = "us-west-2"; + config.disableS3ExpressAuth = false; + + auto endpoint_provider = + Aws::MakeShared("S3ClientFactoryTest"); + auto client = std::make_shared( + std::make_shared(), endpoint_provider, + config); + ASSERT_NE(client, nullptr); + + Aws::S3::Model::GetObjectRequest request; + request.SetBucket("analytics--usw2-az1--x-s3"); + request.SetKey("data.parquet"); + auto outcome = endpoint_provider->ResolveEndpoint(request.GetEndpointContextParams()); + ASSERT_TRUE(outcome.IsSuccess()); + const auto& endpoint = outcome.GetResult(); + EXPECT_EQ(endpoint.GetURI().GetAuthority(), + "analytics--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com"); + ASSERT_TRUE(endpoint.GetAttributes().has_value()); + const auto& attributes = *endpoint.GetAttributes(); + EXPECT_EQ(attributes.backend, "S3Express"); + EXPECT_EQ(attributes.authScheme.GetName(), "S3ExpressSigner"); + ASSERT_TRUE(attributes.authScheme.GetSigningName().has_value()); + EXPECT_EQ(*attributes.authScheme.GetSigningName(), "s3express"); + ASSERT_TRUE(attributes.authScheme.GetSigningRegion().has_value()); + EXPECT_EQ(*attributes.authScheme.GetSigningRegion(), "us-west-2"); } TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) { diff --git a/be/test/util/s3_util_test.cpp b/be/test/util/s3_util_test.cpp index 0fe480df0dde07..a87d0e9a2d2980 100644 --- a/be/test/util/s3_util_test.cpp +++ b/be/test/util/s3_util_test.cpp @@ -77,36 +77,6 @@ TEST_F(S3UTILTest, hide_access_key_typical_aws_key) { EXPECT_EQ("xxxxxxxFODNN7xxxxxxx", result); } -TEST_F(S3UTILTest, is_s3_express_context) { - EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com", - "us-east-1")); - EXPECT_TRUE(is_s3_express("bucket--use1-az4--x-s3", "s3.us-east-1.amazonaws.com", "us-east-1")); - EXPECT_TRUE(is_s3_express("bucket--cnn1-az1--x-s3", "https://s3.cn-north-1.amazonaws.com.cn", - "cn-north-1")); - - // S3-compatible services must retain their existing endpoint and checksum behavior, - // even if a bucket happens to use the Directory Bucket suffix. - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://minio.example.com", "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", - "https://s3.us-east-1.amazonaws.com.example.com", "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--x-s3", "https://s3.us-east-1.amazonaws.com", "us-east-1")); - EXPECT_FALSE( - is_s3_express("bucket--zone--x-s3", "https://s3.us-east-1.amazonaws.com", "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket", "https://example.com/s3express/path", "us-east-1")); - - // Users configure the standard HTTPS regional endpoint. The SDK derives the zonal endpoint. - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "http://s3.us-east-1.amazonaws.com", - "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", "https://s3.us-east-1.amazonaws.com", - "us-west-2")); - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", - "https://s3express-control.us-east-1.amazonaws.com", "us-east-1")); - EXPECT_FALSE(is_s3_express("bucket--use1-az4--x-s3", - "https://bucket--use1-az4--x-s3.s3express-use1-az4.us-east-1." - "amazonaws.com", - "us-east-1")); -} - // Verifies that check_s3_rate_limiter_config_changed() rebuilds the global GET rate // limiter when the related configs change. This is the behavior the cloud vault refresh // thread relies on to apply dynamically modified s3_get_* rate limiter configs without diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index 02f8fe532cd567..b2514b44d0f0e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -22,6 +22,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.datasource.property.storage.BrokerProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.foundation.property.StoragePropertiesException; import org.apache.doris.persist.gson.GsonUtils; @@ -118,6 +119,19 @@ public BrokerDesc(String name, StorageBackend.StorageType storageType, Map properties) { + BrokerDesc brokerDesc = new BrokerDesc(); + brokerDesc.name = name; + brokerDesc.storageType = StorageType.S3; + brokerDesc.properties = Maps.newHashMap(); + if (properties != null) { + brokerDesc.properties.putAll(properties); + } + brokerDesc.storageProperties = S3Properties.createForS3ExpressImport(brokerDesc.properties); + return brokerDesc; + } + public String getFileLocation(String location) throws UserException { return (null != storageProperties) ? storageProperties.validateAndNormalizeUri(location) : location; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java index 63ab2081110409..a2263c2ae56c63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3URI.java @@ -359,25 +359,14 @@ public static boolean isS3DirectoryBucket(final String bucketName) { if (bucketName == null || !bucketName.endsWith(S3_DIRECTORY_BUCKET_SUFFIX)) { return false; } - // Check if the bucket name has the correct format: bucket-name--azid--x-s3 - // The bucket name should have at least 3 segments separated by "--" - String[] segments = bucketName.split("--"); - if (segments.length < 3) { + String nameAndZone = bucketName.substring( + 0, bucketName.length() - S3_DIRECTORY_BUCKET_SUFFIX.length()); + int zoneSeparator = nameAndZone.lastIndexOf("--"); + if (zoneSeparator <= 0 || zoneSeparator + 2 == nameAndZone.length()) { return false; } - // The last segment should be "x-s3" - if (!"x-s3".equals(segments[segments.length - 1])) { - return false; - } - // The second-to-last segment should be the availability zone identifier - // It should have a format like "usw2-az1", "use1-az4", etc. - String azid = segments[segments.length - 2]; - if (azid == null || azid.isEmpty()) { - return false; - } - // Basic validation: azid should contain at least one hyphen and not be empty - // More sophisticated validation could be added here if needed - return azid.contains("-") && azid.length() > 3; + String zoneId = nameAndZone.substring(zoneSeparator + 2); + return zoneId.matches("[a-z0-9-]+-az[0-9]+"); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java index f2de61b671d912..887bcf479f7d75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java @@ -127,10 +127,16 @@ private Map doBuildS3Configuration(String maxConnections, return s3Props; } - public void enableS3ExpressImportRead() { + void enableS3ExpressImportRead() { s3ExpressImportRead = true; } + /** Returns whether the typed properties carry the trusted S3 Express import marker. */ + public static boolean isS3ExpressImportRead(StorageProperties properties) { + return properties instanceof AbstractS3CompatibleProperties + && ((AbstractS3CompatibleProperties) properties).s3ExpressImportRead; + } + protected String getAwsCredentialsProviderTypeForBackend() { if (StringUtils.isBlank(getAccessKey()) && StringUtils.isBlank(getSecretKey())) { return AwsCredentialsProviderMode.ANONYMOUS.name(); @@ -158,19 +164,24 @@ public AwsCredentialsProvider getAwsCredentialsProvider() { @Override public void initNormalizeAndCheckProps() { super.initNormalizeAndCheckProps(); - setEndpointIfPossible(); - setRegionIfPossible(); + // S3 Express imports use the directory bucket name and SDK endpoint rules. Keep the + // user endpoint and region untouched: the object client ignores endpoint, and uses + // region only as a fallback when the bucket Zone ID is unknown to Doris. + if (!s3ExpressImportRead) { + setEndpointIfPossible(); + setRegionIfPossible(); + } //Allow anonymous access if both access_key and secret_key are empty //But not recommended for production use. if (StringUtils.isBlank(getAccessKey()) != StringUtils.isBlank(getSecretKey())) { throw new IllegalArgumentException("Both the access key and the secret key must be set."); } - if (StringUtils.isBlank(getRegion())) { + if (!s3ExpressImportRead && StringUtils.isBlank(getRegion())) { throw new IllegalArgumentException("Region is not set. If you are using a standard endpoint, the region " + "will be detected automatically. Otherwise, please specify it explicitly." ); } - if (StringUtils.isBlank(getEndpoint())) { + if (!s3ExpressImportRead && StringUtils.isBlank(getEndpoint())) { throw new IllegalArgumentException("Endpoint is not set. Please specify it explicitly." ); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 01c5cc6033c3af..36bcdfe53f2bde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -22,6 +22,8 @@ import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.S3URI; import org.apache.doris.datasource.property.common.AwsCredentialsProviderFactory; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; import org.apache.doris.foundation.property.ConnectorPropertiesUtils; @@ -63,6 +65,7 @@ public class S3Properties extends AbstractS3CompatibleProperties { public static final String ROLE_ARN = "s3.role_arn"; public static final String EXTERNAL_ID = "s3.external_id"; public static final String CREDENTIALS_PROVIDER_TYPE = "s3.credentials_provider_type"; + public static final String PROVIDER = "s3.provider"; private static final String[] ENDPOINT_NAMES_FOR_GUESSING = { "s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT", "aws.endpoint", "glue.endpoint", @@ -190,6 +193,11 @@ public class S3Properties extends AbstractS3CompatibleProperties { @Getter private AwsCredentialsProviderMode awsCredentialsProviderMode; + @Getter + @ConnectorProperty(names = {PROVIDER, "provider"}, required = false, + description = "The S3 service provider.") + private String s3Provider = ""; + public static S3Properties of(Map properties) { S3Properties propertiesObj = new S3Properties(properties); ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj, properties); @@ -197,6 +205,47 @@ public static S3Properties of(Map properties) { return propertiesObj; } + /** Creates the trusted S3 properties used only by the scoped S3 import paths. */ + public static S3Properties createForS3ExpressImport(Map properties) { + S3Properties propertiesObj = new S3Properties(properties); + propertiesObj.enableS3ExpressImportRead(); + propertiesObj.initNormalizeAndCheckProps(); + return propertiesObj; + } + + /** Returns whether the user explicitly selected AWS as the S3 provider. */ + public static boolean isAwsProvider(Map properties) { + return properties.entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(PROVIDER) + || entry.getKey().equalsIgnoreCase("provider")) + .map(Map.Entry::getValue) + .anyMatch(value -> "AWS".equalsIgnoreCase(value)); + } + + /** Returns whether a scoped S3 TVF request can use SDK-managed S3 Express access. */ + public static boolean isS3ExpressImport(Map properties) { + if (!isAwsProvider(properties)) { + return false; + } + Optional uri = properties.entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase("uri")) + .map(Map.Entry::getValue) + .findFirst(); + if (uri.isEmpty()) { + return false; + } + return isS3ExpressUri(uri.get()); + } + + /** Returns whether the URI contains a complete S3 Express directory bucket name. */ + public static boolean isS3ExpressUri(String uri) { + try { + return S3URI.create(uri).useS3DirectoryBucket(); + } catch (UserException e) { + return false; + } + } + /** * Pattern to match various AWS S3 endpoint formats and extract the region part. *

    @@ -205,20 +254,15 @@ public static S3Properties of(Map properties) { * - s3.dualstack.us-east-1.amazonaws.com => region = us-east-1 * - s3-fips.us-east-2.amazonaws.com => region = us-east-2 * - s3-fips.dualstack.us-east-2.amazonaws.com => region = us-east-2 - * - s3express-control.us-west-2.amazonaws.com => region = us-west-2 (S3 Directory Bucket Regional) - * - s3express-usw2-az1.us-west-2.amazonaws.com => region = us-west-2 (S3 Directory Bucket Zonal) *

    - * Group(1), Group(2), or Group(3) in the pattern captures the region part if available. + * Group(1) captures the region part if available. *

    * For Glue https://docs.aws.amazon.com/general/latest/gr/glue.html */ private static final Set ENDPOINT_PATTERN = ImmutableSet.of( Pattern.compile( - "^(?:https?://)?(?:" - + "s3(?:[-.]fips)?(?:[-.]dualstack)?[-.]([a-z0-9-]+)|" // Standard S3 endpoints - + "s3express-control\\.([a-z0-9-]+)|" // Directory bucket regional - + "s3express-[a-z0-9-]+\\.([a-z0-9-]+)" // Directory bucket zonal - + ")\\.amazonaws\\.com(?:/.*)?$", + "^(?:https?://)?s3(?:[-.]fips)?(?:[-.]dualstack)?[-.]([a-z0-9-]+)" + + "\\.amazonaws\\.com(?:/.*)?$", Pattern.CASE_INSENSITIVE), Pattern.compile( "^(?:https?://)?glue(?:-fips)?\\.([a-z0-9-]+)\\.(amazonaws\\.com(?:\\.cn)?|api\\.aws)$", @@ -295,6 +339,10 @@ protected Set schemas() { public Map getBackendConfigProperties() { Map backendProperties = generateBackendS3Configuration(); + if (isS3ExpressImportRead(this) && StringUtils.isNotBlank(s3Provider)) { + backendProperties.put("provider", s3Provider); + } + if (StringUtils.isNotBlank(s3IAMRole)) { backendProperties.put("AWS_ROLE_ARN", s3IAMRole); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java index 955e27f9cf99a2..d5b1882495c80d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemFactory.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.common.util.NetUtils; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.BrokerProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.filesystem.spi.FileSystemProvider; @@ -84,6 +85,15 @@ public static void initPluginManager(FileSystemPluginManager manager) { */ public static org.apache.doris.filesystem.FileSystem getFileSystem(Map properties) throws IOException { + // A raw map can come directly from user-managed resources or catalogs. Only the typed + // StorageProperties overload below may carry this import-scope marker into the S3 plugin. + Map untrustedProperties = new HashMap<>(properties); + untrustedProperties.remove(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ); + return getFileSystemInternal(untrustedProperties); + } + + private static org.apache.doris.filesystem.FileSystem getFileSystemInternal(Map properties) + throws IOException { FileSystemPluginManager mgr = pluginManager; if (mgr != null) { return mgr.createFileSystem(properties); @@ -112,7 +122,7 @@ public static org.apache.doris.filesystem.FileSystem getFileSystem(Map fileGroup.getFilePaths().stream()) + .noneMatch(S3Properties::isS3ExpressUri)) { return brokerDesc; } - Map properties = new HashMap<>(brokerDesc.getProperties()); - BrokerDesc importBrokerDesc = new BrokerDesc( - brokerDesc.getName(), brokerDesc.getStorageType(), properties); - if (importBrokerDesc.getStorageProperties() instanceof AbstractS3CompatibleProperties) { - ((AbstractS3CompatibleProperties) importBrokerDesc.getStorageProperties()) - .enableS3ExpressImportRead(); - } - return importBrokerDesc; + return BrokerDesc.createForS3ExpressImport(brokerDesc.getName(), brokerDesc.getProperties()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index c470bc789d9e78..b9186781ef458c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -64,6 +64,7 @@ import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.datasource.FileCacheAdmissionManager; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.dictionary.LayoutType; import org.apache.doris.info.TableRefInfo; import org.apache.doris.info.TableValuedFunctionRefInfo; @@ -2168,12 +2169,19 @@ public Set visitPropertyKeyClause(DorisParser.PropertyKeyClauseContext c @Override public BrokerDesc visitWithRemoteStorageSystem(WithRemoteStorageSystemContext ctx) { + return visitWithRemoteStorageSystem(ctx, false); + } + + private BrokerDesc visitWithRemoteStorageSystem(WithRemoteStorageSystemContext ctx, + boolean s3ExpressImportRead) { BrokerDesc brokerDesc = null; Map brokerPropertiesMap = visitPropertyItemList(ctx.brokerProperties); if (ctx.S3() != null) { - brokerDesc = new BrokerDesc("S3", StorageBackend.StorageType.S3, brokerPropertiesMap); + brokerDesc = s3ExpressImportRead && S3Properties.isAwsProvider(brokerPropertiesMap) + ? BrokerDesc.createForS3ExpressImport("S3", brokerPropertiesMap) + : new BrokerDesc("S3", StorageBackend.StorageType.S3, brokerPropertiesMap); } else if (ctx.HDFS() != null) { brokerDesc = new BrokerDesc("HDFS", StorageBackend.StorageType.HDFS, brokerPropertiesMap); } else if (ctx.LOCAL() != null) { @@ -2225,7 +2233,13 @@ public List> visitMultiStatements(MultiState public LogicalPlan visitLoad(DorisParser.LoadContext ctx) { BrokerDesc brokerDesc = null; if (ctx.withRemoteStorageSystem() != null) { - brokerDesc = visitWithRemoteStorageSystem(ctx.withRemoteStorageSystem()); + boolean s3ExpressImportRead = ctx.dataDescs.stream() + .flatMap(dataDesc -> dataDesc.filePaths.stream()) + .map(Token::getText) + .map(path -> path.substring(1, path.length() - 1)) + .anyMatch(S3Properties::isS3ExpressUri); + brokerDesc = visitWithRemoteStorageSystem( + ctx.withRemoteStorageSystem(), s3ExpressImportRead); } ResourceDesc resourceDesc = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java index 117f0dd3bfb802..2fdfc4d6e39d47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java @@ -34,6 +34,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.S3Util; import org.apache.doris.common.util.TimeUtils; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.ObjectStorageProperties; import org.apache.doris.load.EtlJobType; import org.apache.doris.load.LoadJobRowResult; @@ -457,7 +458,9 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception { } else if (brokerDesc != null) { etlJobType = EtlJobType.BROKER; if (brokerDesc.getFileType() != null && brokerDesc.getFileType().equals(TFileType.FILE_S3) - && brokerDesc.getStorageProperties() instanceof ObjectStorageProperties) { + && brokerDesc.getStorageProperties() instanceof ObjectStorageProperties + && !AbstractS3CompatibleProperties.isS3ExpressImportRead( + brokerDesc.getStorageProperties())) { //@zykkk todo We should use a unified connectivity check — it doesn’t really belong here. ObjectStorageProperties storageProperties = (ObjectStorageProperties) brokerDesc.getStorageProperties(); String endpoint = storageProperties.getEndpoint(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 6c7962e7267033..06d2c35f064aa1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -46,6 +46,7 @@ import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.datasource.property.fileformat.TextFileFormatProperties; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.ObjectStorageProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.datasource.tvf.source.TVFScanNode; @@ -161,7 +162,8 @@ protected void parseFile() throws AnalysisException { BrokerDesc brokerDesc = getBrokerDesc(); try { StorageProperties sp = brokerDesc.getStorageProperties(); - if (sp instanceof ObjectStorageProperties) { + if (sp instanceof ObjectStorageProperties + && !AbstractS3CompatibleProperties.isS3ExpressImportRead(sp)) { S3Util.validateAndTestEndpoint(((ObjectStorageProperties) sp).getEndpoint()); } try (org.apache.doris.filesystem.FileSystem fs = FileSystemFactory.getFileSystem(brokerDesc)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index c21568eb2cf283..35f1dca5a3bdf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -21,7 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; -import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TFileType; @@ -52,9 +52,10 @@ public S3TableValuedFunction(Map properties, boolean s3ExpressIm // 1. analyze common properties Map props = super.parseCommonProperties(properties); try { - this.storageProperties = StorageProperties.createPrimary(props); - if (s3ExpressImportRead) { - enableS3ExpressImportRead(storageProperties); + if (s3ExpressImportRead && S3Properties.isS3ExpressImport(props)) { + this.storageProperties = S3Properties.createForS3ExpressImport(props); + } else { + this.storageProperties = StorageProperties.createPrimary(props); } this.backendConnectProperties.putAll(storageProperties.getBackendConfigProperties()); String uri = storageProperties.validateAndGetUri(props); @@ -87,17 +88,10 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - BrokerDesc brokerDesc = new BrokerDesc("S3TvfBroker", processedParams); - if (s3ExpressImportRead) { - enableS3ExpressImportRead(brokerDesc.getStorageProperties()); - } - return brokerDesc; - } - - private static void enableS3ExpressImportRead(StorageProperties properties) { - if (properties instanceof AbstractS3CompatibleProperties) { - ((AbstractS3CompatibleProperties) properties).enableS3ExpressImportRead(); + if (s3ExpressImportRead && S3Properties.isS3ExpressImport(processedParams)) { + return BrokerDesc.createForS3ExpressImport("S3TvfBroker", processedParams); } + return new BrokerDesc("S3TvfBroker", processedParams); } // =========== implement abstract methods of TableValuedFunctionIf ================= diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3URITest.java b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3URITest.java index 5cfd889ab97f90..0508beb9627c04 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3URITest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3URITest.java @@ -235,6 +235,7 @@ public void testS3DirectoryBucket() throws UserException { Assert.assertFalse(S3URI.isS3DirectoryBucket("my-bucket--usw2-az1--x-s4")); // wrong suffix Assert.assertFalse(S3URI.isS3DirectoryBucket("my-bucket-usw2-az1--x-s3")); // incorrect format Assert.assertFalse(S3URI.isS3DirectoryBucket("my-bucket--usw2az1--x-s3")); // azid without hyphen + Assert.assertFalse(S3URI.isS3DirectoryBucket("my-bucket--usw2-azx--x-s3")); // invalid az number Assert.assertFalse(S3URI.isS3DirectoryBucket("my-bucket---x-s3")); // empty azid Assert.assertFalse(S3URI.isS3DirectoryBucket(null)); Assert.assertFalse(S3URI.isS3DirectoryBucket("")); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index edd45643ba6db0..be3b6022b3039f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -386,36 +386,6 @@ public void testGetAwsCredentialsProviderWithAccessKeyAndSecretKey() throws User Assertions.assertTrue(provider instanceof StaticCredentialsProvider); } - @Test - public void testS3ExpressEndpointPattern() throws UserException { - origProps.put("s3.access_key", "myS3AccessKey"); - origProps.put("s3.secret_key", "myS3SecretKey"); - - // S3 Express Control Endpoint (Regional) - String endpointControl = "s3express-control.us-west-2.amazonaws.com"; - origProps.put("s3.endpoint", endpointControl); - S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); - Assertions.assertEquals("us-west-2", s3Properties.getRegion()); - - // S3 Express Zonal Endpoint - String endpointZonal = "s3express-usw2-az1.us-west-2.amazonaws.com"; - origProps.put("s3.endpoint", endpointZonal); - s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); - Assertions.assertEquals("us-west-2", s3Properties.getRegion()); - - // Test with https scheme - String endpointWithScheme = "https://s3express-control.eu-central-1.amazonaws.com"; - origProps.put("s3.endpoint", endpointWithScheme); - s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); - Assertions.assertEquals("eu-central-1", s3Properties.getRegion()); - - // Test with path - String endpointWithPath = "https://s3express-control.eu-central-1.amazonaws.com/path/to/obj"; - origProps.put("s3.endpoint", endpointWithPath); - s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); - Assertions.assertEquals("eu-central-1", s3Properties.getRegion()); - } - @Test public void testInvalidEndpoint() { origProps.put("s3.access_key", "myS3AccessKey"); @@ -559,14 +529,35 @@ public void testS3DisableHadoopCache() throws UserException { public void testS3ExpressImportMarkerRequiresInternalOptIn() { origProps.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com"); origProps.put("s3.region", "us-west-2"); + origProps.put("s3.provider", "AWS"); origProps.put(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ, "true"); S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(origProps); Assertions.assertFalse(s3Properties.getBackendConfigProperties() .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assertions.assertFalse(s3Properties.getBackendConfigProperties().containsKey("provider")); s3Properties.enableS3ExpressImportRead(); Assertions.assertEquals("true", s3Properties.getBackendConfigProperties() .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assertions.assertEquals("AWS", s3Properties.getBackendConfigProperties().get("provider")); + } + + @Test + public void testS3ExpressImportAllowsSdkEndpointResolution() { + origProps.put("uri", "s3://analytics--usw2-az1--x-s3/data/file.parquet"); + origProps.put("s3.provider", "AWS"); + origProps.put("s3.access_key", "myS3AccessKey"); + origProps.put("s3.secret_key", "myS3SecretKey"); + + Assertions.assertTrue(S3Properties.isS3ExpressImport(origProps)); + S3Properties s3Properties = S3Properties.createForS3ExpressImport(origProps); + Assertions.assertTrue(AbstractS3CompatibleProperties.isS3ExpressImportRead(s3Properties)); + Map backendProperties = s3Properties.getBackendConfigProperties(); + Assertions.assertEquals("true", + backendProperties.get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assertions.assertEquals("AWS", backendProperties.get("provider")); + Assertions.assertEquals("", backendProperties.get("AWS_ENDPOINT")); + Assertions.assertEquals("", backendProperties.get("AWS_REGION")); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java new file mode 100644 index 00000000000000..7e8004385b4766 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.doris.fs; + +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; +import org.apache.doris.datasource.property.storage.S3Properties; +import org.apache.doris.filesystem.FileSystem; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; + +public class FileSystemFactoryS3ExpressScopeTest { + + @After + public void tearDown() { + FileSystemFactory.clearProviderCache(); + } + + @Test + public void testRawMapCannotInjectS3ExpressImportMarker() throws Exception { + FileSystemPluginManager manager = Mockito.mock(FileSystemPluginManager.class); + Mockito.when(manager.createFileSystem(Mockito.anyMap())).thenReturn(Mockito.mock(FileSystem.class)); + FileSystemFactory.initPluginManager(manager); + Map rawProperties = new HashMap<>(); + rawProperties.put(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ, "true"); + + FileSystemFactory.getFileSystem(rawProperties); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + Mockito.verify(manager).createFileSystem(captor.capture()); + Assert.assertFalse(captor.getValue() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertTrue(rawProperties + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + } + + @Test + public void testTypedS3ExpressImportKeepsTrustedMarker() throws Exception { + FileSystemPluginManager manager = Mockito.mock(FileSystemPluginManager.class); + Mockito.when(manager.createFileSystem(Mockito.anyMap())).thenReturn(Mockito.mock(FileSystem.class)); + FileSystemFactory.initPluginManager(manager); + Map properties = new HashMap<>(); + properties.put("uri", "s3://analytics--usw2-az1--x-s3/data.parquet"); + properties.put("s3.provider", "AWS"); + properties.put("s3.access_key", "ak"); + properties.put("s3.secret_key", "sk"); + + FileSystemFactory.getFileSystem(S3Properties.createForS3ExpressImport(properties)); + + ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + Mockito.verify(manager).createFileSystem(captor.capture()); + Assert.assertEquals("true", captor.getValue() + .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java index a304be9ff8744d..825823f57695db 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/BrokerLoadJobTest.java @@ -73,11 +73,20 @@ public static void start() { @Test public void testS3ExpressImportPropertyIsTaskScopedToBrokerLoad() { Map properties = Maps.newHashMap(); - properties.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com"); - properties.put("s3.region", "us-west-2"); + properties.put("s3.provider", "AWS"); + properties.put("s3.endpoint", "https://endpoint-is-ignored.example.com"); + properties.put("s3.region", "us-east-1"); BrokerDesc original = new BrokerDesc("S3", StorageBackend.StorageType.S3, properties); BrokerLoadJob brokerLoadJob = new BrokerLoadJob(); Deencapsulation.setField(brokerLoadJob, "brokerDesc", original); + BrokerFileGroup expressFileGroup = Mockito.mock(BrokerFileGroup.class); + Mockito.when(expressFileGroup.getFilePaths()).thenReturn( + Collections.singletonList("s3://analytics--usw2-az1--x-s3/data.parquet")); + BrokerFileGroupAggInfo fileGroupAggInfo = new BrokerFileGroupAggInfo(); + Deencapsulation.setField(fileGroupAggInfo, "aggKeyToFileGroups", + Collections.singletonMap(new FileGroupAggKey(1L, null), + Collections.singletonList(expressFileGroup))); + Deencapsulation.setField(brokerLoadJob, "fileGroupAggInfo", fileGroupAggInfo); BrokerDesc taskBrokerDesc = brokerLoadJob.brokerDescForS3ExpressImport(); @@ -87,6 +96,12 @@ public void testS3ExpressImportPropertyIsTaskScopedToBrokerLoad() { Assert.assertEquals("true", taskBrokerDesc.getBackendConfigProperties() .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Mockito.when(expressFileGroup.getFilePaths()).thenReturn( + Collections.singletonList("s3://ordinary-bucket/data.parquet")); + Assert.assertSame(original, brokerLoadJob.brokerDescForS3ExpressImport()); + + Mockito.when(expressFileGroup.getFilePaths()).thenReturn( + Collections.singletonList("s3://analytics--usw2-az1--x-s3/data.parquet")); Deencapsulation.setField(brokerLoadJob, "jobType", EtlJobType.COPY); Assert.assertSame(original, brokerLoadJob.brokerDescForS3ExpressImport()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java index 60ee030dcf3624..b1ae621b0632ec 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java @@ -22,6 +22,7 @@ import org.apache.doris.common.Pair; import org.apache.doris.datasource.property.fileformat.CsvFileFormatProperties; import org.apache.doris.datasource.property.fileformat.DeferredFileFormatProperties; +import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.load.NereidsDataDescription; @@ -145,6 +146,35 @@ public void testLoadCommandWithMultipleFiles() { Assertions.assertEquals("s3://bucket/customer/part-3", filePaths.get(2)); } + @Test + public void testS3ExpressLoadWithoutEndpointAndRegion() { + String loadSql = "LOAD LABEL customer_s3_express_test( " + + " DATA INFILE(\"s3://analytics--usw2-az1--x-s3/customer/*.parquet\") " + + " INTO TABLE customer" + + " ) " + + " WITH S3( " + + " \"s3.provider\" = \"AWS\", " + + " \"s3.access_key\" = \"AK\", " + + " \"s3.secret_key\" = \"SK\", " + + " \"use_path_style\" = \"false\");"; + + List> statements = new NereidsParser().parseMultiple(loadSql); + Assertions.assertFalse(statements.isEmpty()); + + LoadCommand command = (LoadCommand) statements.get(0).first; + Map backendProperties = command.getBrokerDesc().getBackendConfigProperties(); + Assertions.assertEquals("true", + backendProperties.get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assertions.assertEquals("AWS", backendProperties.get("provider")); + Assertions.assertEquals("", backendProperties.get("AWS_ENDPOINT")); + Assertions.assertEquals("", backendProperties.get("AWS_REGION")); + + String ordinaryBucketSql = loadSql.replace( + "analytics--usw2-az1--x-s3", "ordinary-bucket"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> new NereidsParser().parseMultiple(ordinaryBucketSql)); + } + @Test public void testLoadCommand() throws Exception { String loadSql1 = "LOAD LABEL customer_lable_for_test( " diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index dd7f5cca53a1d7..37b726aa8eb6cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -132,6 +132,7 @@ public void testS3ExpressMarkerIsScopedToOneShotInsertStatement() throws Analysi try { Map properties = Maps.newHashMap(); properties.put("uri", "s3://analytics--usw2-az1--x-s3/data/file.csv"); + properties.put("s3.provider", "AWS"); properties.put("s3.endpoint", "https://s3.us-west-2.amazonaws.com"); properties.put("s3.region", "us-west-2"); properties.put("format", "csv"); @@ -154,7 +155,11 @@ public void testS3ExpressMarkerIsScopedToOneShotInsertStatement() throws Analysi .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); statementContext.setS3ExpressImportRead(true); - S3TableValuedFunction insertTvf = (S3TableValuedFunction) new S3(new Properties(properties)) + Map recommendedProperties = Maps.newHashMap(properties); + recommendedProperties.remove("s3.endpoint"); + recommendedProperties.remove("s3.region"); + S3TableValuedFunction insertTvf = (S3TableValuedFunction) new S3( + new Properties(recommendedProperties)) .getCatalogFunction(); Assert.assertEquals("true", insertTvf.getBackendConnectProperties() .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); @@ -162,6 +167,8 @@ public void testS3ExpressMarkerIsScopedToOneShotInsertStatement() throws Analysi .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); Assert.assertFalse(insertTvf.processedParams .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertEquals("", insertTvf.getBackendConnectProperties().get("AWS_ENDPOINT")); + Assert.assertEquals("", insertTvf.getBackendConnectProperties().get("AWS_REGION")); statementContext.setS3ExpressImportRead(false); S3TableValuedFunction streamingTvf = (S3TableValuedFunction) new S3(new Properties(properties)) diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java index e1b1ea98eb6a33..bb510f6de9a09e 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystem.java @@ -53,7 +53,7 @@ public Optional properties() { @Override protected String globListPrefix(String bucket, String globPattern) throws IOException { - if (isDirectoryBucketEndpoint() || s3ObjStorage.usesS3ExpressRead(bucket)) { + if (s3ObjStorage.usesS3ExpressRead(bucket)) { return slashTerminatedNonGlobPrefix(globPattern); } return super.globListPrefix(bucket, globPattern); @@ -62,16 +62,12 @@ protected String globListPrefix(String bucket, String globPattern) throws IOExce @Override protected List globListPrefixes(String bucket, String globPattern, String listPrefix) throws IOException { - if (isDirectoryBucketEndpoint() || s3ObjStorage.usesS3ExpressRead(bucket)) { + if (s3ObjStorage.usesS3ExpressRead(bucket)) { return List.of(listPrefix); } return super.globListPrefixes(bucket, globPattern, listPrefix); } - private boolean isDirectoryBucketEndpoint() { - return properties != null && properties.isDirectoryBucketEndpoint(); - } - private static String slashTerminatedNonGlobPrefix(String globPattern) { String prefix = longestNonGlobPrefix(globPattern); if (prefix.isEmpty() || prefix.endsWith("/")) { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java index da0d0752ac3e7b..8670ce89224834 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProperties.java @@ -63,6 +63,8 @@ public final class S3FileSystemProperties public static final String CONNECTION_TIMEOUT_MS = "s3.connection.timeout"; public static final String USE_PATH_STYLE = "use_path_style"; public static final String CREDENTIALS_PROVIDER_TYPE = "s3.credentials_provider_type"; + private static final String S3_PROVIDER = "s3.provider"; + private static final String PROVIDER = "provider"; // Must match AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ and the BE property key. static final String S3_EXPRESS_IMPORT_READ = "__DORIS_S3_EXPRESS_IMPORT_READ__"; @@ -74,11 +76,8 @@ public final class S3FileSystemProperties private static final Pattern[] ENDPOINT_PATTERNS = new Pattern[] { Pattern.compile( - "^(?:https?://)?(?:" - + "s3(?:[-.]fips)?(?:[-.]dualstack)?[-.]([a-z0-9-]+)|" - + "s3express-control\\.([a-z0-9-]+)|" - + "s3express-[a-z0-9-]+\\.([a-z0-9-]+)" - + ")\\.amazonaws\\.com(?:/.*)?$", + "^(?:https?://)?s3(?:[-.]fips)?(?:[-.]dualstack)?[-.]([a-z0-9-]+)" + + "\\.amazonaws\\.com(?:/.*)?$", Pattern.CASE_INSENSITIVE), Pattern.compile( "^(?:https?://)?glue(?:-fips)?\\.([a-z0-9-]+)\\.(amazonaws\\.com(?:\\.cn)?|api\\.aws)$", @@ -188,7 +187,9 @@ private S3FileSystemProperties(Map rawProperties) { this.rawProperties = Collections.unmodifiableMap(new HashMap<>(rawProperties)); this.matchedProperties = Collections.unmodifiableMap(collectMatchedProperties(rawProperties)); ConnectorPropertiesUtils.bindConnectorProperties(this, rawProperties); - normalizeForLegacyS3Compatibility(); + if (!isScopedAwsS3ExpressImport()) { + normalizeForLegacyS3Compatibility(); + } } /** Binds and validates raw properties. */ @@ -209,7 +210,8 @@ public void validate() { "s3.external_id must be used together with s3.role_arn") .check(this::hasUnsupportedCredentialsProviderType, "Unsupported s3.credentials_provider_type: " + credentialsProviderType) - .check(() -> StringUtils.isBlank(endpoint) && StringUtils.isBlank(region), + .check(() -> StringUtils.isBlank(endpoint) && StringUtils.isBlank(region) + && !isScopedAwsS3ExpressImport(), "Either s3.endpoint or s3.region must be set") .check(this::hasInvalidUsePathStyle, "use_path_style must be true or false, got: '" + getUsePathStyle() + "'") @@ -335,15 +337,26 @@ public boolean hasAssumeRole() { return StringUtils.isNotBlank(roleArn); } - public boolean isDirectoryBucketEndpoint() { - return StringUtils.containsIgnoreCase(endpoint, "s3express-control.") - || StringUtils.containsIgnoreCase(endpoint, "s3express-"); - } - boolean isS3ExpressImportReadEnabled() { return Boolean.parseBoolean(rawProperties.get(S3_EXPRESS_IMPORT_READ)); } + boolean isAwsProvider() { + return isAwsProvider(rawProperties); + } + + boolean isScopedAwsS3ExpressImport() { + return isS3ExpressImportReadEnabled() && isAwsProvider(); + } + + static boolean isAwsProvider(Map properties) { + return properties.entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(S3_PROVIDER) + || entry.getKey().equalsIgnoreCase(PROVIDER)) + .map(Map.Entry::getValue) + .anyMatch(value -> "AWS".equalsIgnoreCase(value)); + } + private static void putIfNotBlank(Map map, String key, String value) { if (StringUtils.isNotBlank(value)) { map.put(key, value); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java index 1022074791909b..0ae5e8c38fa6fd 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java @@ -61,6 +61,10 @@ public class S3FileSystemProvider implements FileSystemProvider properties) { + if (Boolean.parseBoolean(properties.get(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ)) + && S3FileSystemProperties.isAwsProvider(properties)) { + return true; + } boolean hasCredential = hasAny(properties, ACCESS_KEY_NAMES) || hasAny(properties, ROLE_ARN_NAMES) || hasAny(properties, CREDENTIALS_PROVIDER_TYPE_NAMES); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index 8dc186c8be9d46..cf0b3acafa0f3c 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -76,14 +76,13 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; /** @@ -95,11 +94,14 @@ public class S3ObjStorage implements ObjStorage { private static final Logger LOG = LogManager.getLogger(S3ObjStorage.class); private static final String DIRECTORY_BUCKET_SUFFIX = "--x-s3"; - private static final Pattern DIRECTORY_BUCKET_PATTERN = - Pattern.compile("^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?--[a-z0-9-]+-az[0-9]+--x-s3$"); - private static final Pattern AWS_S3_REGIONAL_ENDPOINT_PATTERN = Pattern.compile( - "^(?:https://)?s3\\.([a-z0-9-]+)\\.amazonaws\\.com(?:\\.cn)?(?::443)?/?$", - Pattern.CASE_INSENSITIVE); + private static final Map S3_EXPRESS_REGIONS = Map.of( + "use1", "us-east-1", + "use2", "us-east-2", + "usw2", "us-west-2", + "aps1", "ap-south-1", + "apne1", "ap-northeast-1", + "euw1", "eu-west-1", + "eun1", "eu-north-1"); /** Validity period for pre-signed URLs and STS tokens (seconds). */ private static final int SESSION_EXPIRE_SECONDS = 3600; @@ -109,9 +111,8 @@ public class S3ObjStorage implements ObjStorage { /** Bucket name; may be null if not provided (listObjectsWithPrefix and related methods will fail). */ private final String bucket; private final AtomicBoolean closed = new AtomicBoolean(false); + private final Map expressClients = new HashMap<>(); private volatile S3Client client; - private volatile S3Client expressClient; - private volatile AwsCredentialsProvider credentialsProvider; public S3ObjStorage(S3FileSystemProperties properties) { this.s3Properties = properties; @@ -156,14 +157,14 @@ protected S3Client buildClient() throws IOException { return buildClient( s3Properties.getEndpoint(), s3Properties.getRegion(), - getCredentialsProvider(), false); + buildCredentialsProvider(), false); } - protected S3Client buildExpressClient() throws IOException { + protected S3Client buildExpressClient(String region) throws IOException { return buildClient( - s3Properties.getEndpoint(), - s3Properties.getRegion(), - getCredentialsProvider(), true); + "", + region, + buildCredentialsProvider(region), true); } private S3Client buildClient(String endpointStr, String region, @@ -180,8 +181,8 @@ private S3Client buildClient(String endpointStr, String region, .chunkedEncodingEnabled(false) .pathStyleAccessEnabled(express ? false : usePathStyle) .build()); - if (s3Properties.isS3ExpressImportReadEnabled() && !s3Properties.isDirectoryBucketEndpoint()) { - builder.disableS3ExpressSessionAuth(!express); + if (express) { + builder.disableS3ExpressSessionAuth(false); } if (!express && StringUtils.isNotBlank(endpointStr)) { @@ -209,60 +210,33 @@ protected AwsCredentialsProvider buildCredentialsProvider() { return S3CredentialsProviderFactory.createClientProvider(s3Properties, this::buildStsClient); } - private AwsCredentialsProvider getCredentialsProvider() { - if (credentialsProvider == null) { - synchronized (this) { - if (credentialsProvider == null) { - credentialsProvider = buildCredentialsProvider(); - } - } - } - return credentialsProvider; + protected AwsCredentialsProvider buildCredentialsProvider(String region) { + return S3CredentialsProviderFactory.createClientProvider( + s3Properties, (sourceCredentials, ignoredUserRegion) -> + buildStsClient(sourceCredentials, region)); } - boolean usesS3ExpressRead(String requestBucket) throws IOException { - if (!s3Properties.isS3ExpressImportReadEnabled() - || requestBucket == null || !requestBucket.endsWith(DIRECTORY_BUCKET_SUFFIX) - || StringUtils.isBlank(s3Properties.getEndpoint())) { - return false; - } - Matcher endpoint = AWS_S3_REGIONAL_ENDPOINT_PATTERN.matcher(s3Properties.getEndpoint()); - if (!endpoint.matches()) { - if (s3Properties.isDirectoryBucketEndpoint()) { - return false; - } - if (isAwsEndpoint(s3Properties.getEndpoint())) { - throw new IOException("AWS Directory Bucket requires a standard HTTPS regional S3 " - + "endpoint; do not use s3express-control or a zonal endpoint"); - } - return false; - } - if (!isDirectoryBucketName(requestBucket)) { - throw new IOException("Invalid AWS Directory Bucket name " + requestBucket - + "; expected ----x-s3"); - } - if (!endpoint.group(1).equalsIgnoreCase(s3Properties.getRegion())) { - throw new IOException("AWS Directory Bucket endpoint region " + endpoint.group(1) - + " does not match configured region " + s3Properties.getRegion()); - } - if (usePathStyle) { - throw new IOException("Path-style addressing is not supported for AWS Directory Bucket"); - } - return true; + boolean usesS3ExpressRead(String requestBucket) { + return s3Properties.isScopedAwsS3ExpressImport() + && getS3ExpressZoneId(requestBucket).isPresent(); } - private S3Client getExpressClient() throws IOException { + private S3Client getExpressClient(String requestBucket) throws IOException { if (closed.get()) { throw new IOException("S3ObjStorage is already closed"); } - if (expressClient == null) { - synchronized (this) { - if (expressClient == null) { - expressClient = buildExpressClient(); - } + String region = getS3ExpressRegion(requestBucket); + synchronized (expressClients) { + if (closed.get()) { + throw new IOException("S3ObjStorage is already closed"); + } + S3Client expressClient = expressClients.get(region); + if (expressClient == null) { + expressClient = buildExpressClient(region); + expressClients.put(region, expressClient); } + return expressClient; } - return expressClient; } private AwsCredentialsProvider buildStsSourceCredentialsProvider() { @@ -310,7 +284,7 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions } } try { - S3Client listClient = expressRead ? getExpressClient() : getClient(); + S3Client listClient = expressRead ? getExpressClient(uri.bucket()) : getClient(); ListObjectsV2Response response = listClient.listObjectsV2(builder.build()); List objects = response.contents().stream() .map(s3Obj -> new org.apache.doris.filesystem.spi.RemoteObject( @@ -740,8 +714,50 @@ private static String getRelativePathSafe(String prefix, String key) { return key.substring(normalized.length()); } - private static boolean isDirectoryBucketName(String bucketName) { - return bucketName != null && DIRECTORY_BUCKET_PATTERN.matcher(bucketName).matches(); + private static Optional getS3ExpressZoneId(String bucketName) { + if (bucketName == null || !bucketName.endsWith(DIRECTORY_BUCKET_SUFFIX)) { + return Optional.empty(); + } + String nameAndZone = bucketName.substring( + 0, bucketName.length() - DIRECTORY_BUCKET_SUFFIX.length()); + int zoneSeparator = nameAndZone.lastIndexOf("--"); + if (zoneSeparator <= 0 || zoneSeparator + 2 == nameAndZone.length()) { + return Optional.empty(); + } + String zoneId = nameAndZone.substring(zoneSeparator + 2); + int azSeparator = zoneId.lastIndexOf("-az"); + if (azSeparator <= 0 || azSeparator + 3 == zoneId.length()) { + return Optional.empty(); + } + for (int i = 0; i < azSeparator; i++) { + char c = zoneId.charAt(i); + if (!(c >= 'a' && c <= 'z') && !(c >= '0' && c <= '9') && c != '-') { + return Optional.empty(); + } + } + for (int i = azSeparator + 3; i < zoneId.length(); i++) { + char c = zoneId.charAt(i); + if (c < '0' || c > '9') { + return Optional.empty(); + } + } + return Optional.of(zoneId); + } + + private String getS3ExpressRegion(String bucketName) throws IOException { + String zoneId = getS3ExpressZoneId(bucketName).orElseThrow( + () -> new IOException("Invalid AWS Directory Bucket name " + bucketName + + "; expected ----x-s3")); + String zonePrefix = zoneId.substring(0, zoneId.indexOf('-')); + String inferredRegion = S3_EXPRESS_REGIONS.get(zonePrefix); + if (inferredRegion != null) { + return inferredRegion; + } + if (StringUtils.isBlank(s3Properties.getRegion())) { + throw new IOException("Cannot infer AWS region from Directory Bucket zone " + zoneId + + "; set s3.region for newly introduced AWS regions"); + } + return s3Properties.getRegion(); } private static String slashTerminatedPrefix(String key) { @@ -752,19 +768,6 @@ private static String slashTerminatedPrefix(String key) { return slash < 0 ? "" : key.substring(0, slash + 1); } - private static boolean isAwsEndpoint(String endpoint) { - int schemeSeparator = endpoint.indexOf("://"); - URI uri = URI.create(schemeSeparator > 0 ? endpoint : "https://" + endpoint); - String host = uri.getHost(); - if (host == null) { - return false; - } - host = host.toLowerCase(Locale.ROOT); - return host.endsWith(".amazonaws.com") - || host.endsWith(".amazonaws.com.cn") - || host.endsWith(".api.aws"); - } - @Override public void close() throws IOException { if (closed.compareAndSet(false, true)) { @@ -772,9 +775,11 @@ public void close() throws IOException { client.close(); client = null; } - if (expressClient != null) { - expressClient.close(); - expressClient = null; + synchronized (expressClients) { + for (S3Client expressClient : expressClients.values()) { + expressClient.close(); + } + expressClients.clear(); } } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java index fe5c21d9515460..1a27a7f98d43a1 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java @@ -89,6 +89,28 @@ void supports_acceptsExplicitS3SupportWithoutCredentials() { Assertions.assertTrue(provider.supports(props)); } + @Test + void supports_acceptsScopedAwsExpressWithoutEndpointOrRegion() { + Map props = new HashMap<>(); + props.put("s3.provider", "AWS"); + props.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); + + Assertions.assertTrue(provider.supports(props)); + S3FileSystemProperties bound = provider.bind(props); + Assertions.assertTrue(bound.isScopedAwsS3ExpressImport()); + Assertions.assertEquals("", bound.getEndpoint()); + Assertions.assertEquals("", bound.getRegion()); + } + + @Test + void supports_rejectsExpressMarkerWithoutExplicitAwsProvider() { + Map props = new HashMap<>(); + props.put("provider", "S3"); + props.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); + + Assertions.assertFalse(provider.supports(props)); + } + @Test void bind_returnsValidatedS3FileSystemProperties() { Map props = new HashMap<>(); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java index db24f228ff9177..ce2a2a85407f82 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemTest.java @@ -677,33 +677,6 @@ void globListWithLimit_paginatesExpandedPrefixesInUtf8BinaryOrder() throws IOExc Assertions.assertEquals("s3://bucket/" + emojiKey, secondPage.getFiles().get(0).location().uri()); } - @Test - void globListWithLimit_directoryBucketFallsBackToSlashTerminatedStaticPrefix() throws IOException { - S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( - "s3.endpoint", "https://s3express-usw2-az1.us-west-2.amazonaws.com", - "s3.region", "us-west-2")); - S3FileSystem directoryBucketFs = new S3FileSystem(properties, mockStorage); - Mockito.when(mockStorage.listObjects( - ArgumentMatchers.eq("s3://bucket/data/"), ArgumentMatchers.isNull())) - .thenReturn(new RemoteObjects( - List.of( - new RemoteObject("data/a.csv", "a.csv", null, 10L, 0L), - new RemoteObject("data/b.csv", "b.csv", null, 20L, 0L)), - false, null)); - - GlobListing listing = directoryBucketFs.globListWithLimit( - Location.of("s3://bucket/data/[ab]*.csv"), null, 0L, 0L); - - Assertions.assertEquals(2, listing.getFiles().size()); - Assertions.assertEquals("data/", listing.getPrefix()); - Mockito.verify(mockStorage).listObjects( - ArgumentMatchers.eq("s3://bucket/data/"), ArgumentMatchers.isNull()); - Mockito.verify(mockStorage, Mockito.never()).listObjects( - ArgumentMatchers.eq("s3://bucket/data/a"), ArgumentMatchers.any()); - Mockito.verify(mockStorage, Mockito.never()).listObjects( - ArgumentMatchers.eq("s3://bucket/data/b"), ArgumentMatchers.any()); - } - @Test void globListWithLimit_regionalDirectoryBucketUsesSlashTerminatedPrefix() throws IOException { S3FileSystemProperties properties = S3FileSystemProperties.of(Map.of( diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java index ff664c368e442b..c8cf3a0e5d7610 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java @@ -58,13 +58,16 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.time.Instant; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.function.Function; /** * Full unit tests for {@link S3ObjStorage} using a testable subclass that overrides - * {@link S3ObjStorage#buildClient()} and {@link S3ObjStorage#buildExpressClient()} + * {@link S3ObjStorage#buildClient()} and {@link S3ObjStorage#buildExpressClient(String)} * to inject mock S3 clients. */ class S3ObjStorageMockTest { @@ -250,50 +253,130 @@ void listObjects_directoryBucketRejectsStartAfter() { } @Test - void listObjects_directorySuffixOnThirdPartyEndpointUsesRegularClient() throws IOException { + void listObjects_directoryBucketIgnoresConfiguredEndpoint() throws IOException { S3Client regularClient = Mockito.mock(S3Client.class); S3Client expressClient = Mockito.mock(S3Client.class); - S3ObjStorage compatibleStorage = directoryBucketStorage( + S3ObjStorage expressStorage = directoryBucketStorage( "https://minio.example.com", "us-west-2", false, regularClient, expressClient); - Mockito.when(regularClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); - compatibleStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/file", null); + expressStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/file", null); ArgumentCaptor captor = ArgumentCaptor.forClass(ListObjectsV2Request.class); - Mockito.verify(regularClient).listObjectsV2(captor.capture()); - Assertions.assertEquals("data/file", captor.getValue().prefix()); - Mockito.verifyNoInteractions(expressClient); + Mockito.verify(expressClient).listObjectsV2(captor.capture()); + Assertions.assertEquals("data/", captor.getValue().prefix()); + Mockito.verifyNoInteractions(regularClient); } @Test - void usesS3ExpressRead_validatesNativeDirectoryBucketContext() throws IOException { + void usesS3ExpressRead_requiresScopeProviderAndCompleteBucketName() { S3Client regularClient = Mockito.mock(S3Client.class); S3Client expressClient = Mockito.mock(S3Client.class); Assertions.assertTrue(directoryBucketStorage( - "s3.us-west-2.amazonaws.com", "us-west-2", false, - regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); - Assertions.assertFalse(directoryBucketStorage( - "https://minio.example.com", "us-west-2", false, - regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); - Assertions.assertFalse(directoryBucketStorage( - "https://s3express-usw2-az1.us-west-2.amazonaws.com", "us-west-2", false, + "http://ignored.example.com", "wrong-region", true, regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); + Map nonAwsProperties = directoryBucketProperties("", "us-west-2", false); + nonAwsProperties.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); + nonAwsProperties.put("provider", "S3"); + Assertions.assertFalse(new TestableS3ObjStorage( + nonAwsProperties, regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); - Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( - "http://s3.us-west-2.amazonaws.com", "us-west-2", false, - regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); - Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( - "https://s3.us-west-2.amazonaws.com", "us-east-1", false, - regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); - Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( - "https://s3.us-west-2.amazonaws.com", "us-west-2", true, - regularClient, expressClient).usesS3ExpressRead(DIRECTORY_BUCKET)); - Assertions.assertThrows(IOException.class, () -> directoryBucketStorage( - "https://s3.us-west-2.amazonaws.com", "us-west-2", false, - regularClient, expressClient).usesS3ExpressRead("invalid--x-s3")); + S3ObjStorage expressStorage = directoryBucketStorage( + "", "us-west-2", false, regularClient, expressClient); + Assertions.assertTrue(expressStorage.usesS3ExpressRead( + "analytics--archive--usw2-az1--x-s3")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("ordinary-bucket")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("invalid--x-s3")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("--usw2-az1--x-s3")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("analytics----x-s3")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("analytics--usw2-az--x-s3")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("analytics--usw2-azx--x-s3")); + Assertions.assertFalse(expressStorage.usesS3ExpressRead("analytics--USW2-az1--x-s3")); + } + + @Test + void listObjects_knownZonePrefixesOverrideConfiguredRegion() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + TestableS3ObjStorage expressStorage = directoryBucketStorage( + "https://ignored.example.com", "wrong-region", false, + regularClient, expressClient); + Map expectedRegions = Map.of( + "use1", "us-east-1", + "use2", "us-east-2", + "usw2", "us-west-2", + "aps1", "ap-south-1", + "apne1", "ap-northeast-1", + "euw1", "eu-west-1", + "eun1", "eu-north-1"); + + for (String zonePrefix : expectedRegions.keySet()) { + expressStorage.listObjects( + "s3://analytics--" + zonePrefix + "-az1--x-s3/data/file.csv", null); + } + + Assertions.assertEquals( + Set.copyOf(expectedRegions.values()), + Set.copyOf(expressStorage.getBuiltExpressRegions())); + Mockito.verifyNoInteractions(regularClient); + } + + @Test + void listObjects_unknownZonePrefixUsesConfiguredRegionVerbatim() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + TestableS3ObjStorage expressStorage = directoryBucketStorage( + "https://ignored.example.com", "me-central-future-1", false, + regularClient, expressClient); + + expressStorage.listObjects( + "s3://analytics--mec9-az1--x-s3/data/file.csv", null); + + Assertions.assertEquals( + List.of("me-central-future-1"), expressStorage.getBuiltExpressRegions()); + } + + @Test + void listObjects_unknownZonePrefixWithoutConfiguredRegionFails() { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + TestableS3ObjStorage expressStorage = directoryBucketStorage( + "https://ignored.example.com", "", false, + regularClient, expressClient); + + IOException exception = Assertions.assertThrows(IOException.class, + () -> expressStorage.listObjects( + "s3://analytics--mec9-az1--x-s3/data/file.csv", null)); + + Assertions.assertTrue(exception.getMessage().contains("set s3.region")); + Mockito.verifyNoInteractions(regularClient, expressClient); + } + + @Test + void listObjects_knownZoneWithoutEndpointOrRegionCachesClientByEffectiveRegion() throws IOException { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + TestableS3ObjStorage expressStorage = directoryBucketStorage( + "", "", true, regularClient, expressClient); + + expressStorage.listObjects( + "s3://analytics--usw2-az1--x-s3/data/file.csv", null); + expressStorage.listObjects( + "s3://logs--archive--usw2-lax1-az1--x-s3/data/file.csv", null); + + Assertions.assertEquals(List.of("us-west-2"), expressStorage.getBuiltExpressRegions()); + Mockito.verify(expressClient, Mockito.times(2)).listObjectsV2( + ArgumentMatchers.any(ListObjectsV2Request.class)); + Mockito.verifyNoInteractions(regularClient); } // ------------------------------------------------------------------ @@ -610,6 +693,23 @@ void buildCredentialsProvider_returnsAssumeRoleProviderWhenRoleArnConfigured() { credentialsProvider.getClass().getSimpleName()); } + @Test + void buildExpressCredentialsProvider_usesRegionInferredFromBucketForAssumeRole() { + Map props = directoryBucketProperties("", "", false); + props.remove("AWS_ACCESS_KEY"); + props.remove("AWS_SECRET_KEY"); + props.put("AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/MyRole"); + props.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); + InspectableS3ObjStorage roleArnStorage = new InspectableS3ObjStorage(props, mockS3); + + AwsCredentialsProvider credentialsProvider = + roleArnStorage.inspectBuildCredentialsProvider("us-west-2"); + + Assertions.assertEquals("StsAssumeRoleCredentialsProvider", + credentialsProvider.getClass().getSimpleName()); + Assertions.assertEquals(List.of("us-west-2"), roleArnStorage.getBuiltStsRegions()); + } + @Test void buildCredentialsProvider_usesTypedCredentialsProviderType() { Map props = new HashMap<>(); @@ -662,21 +762,28 @@ void close_closesS3Client() throws IOException { } @Test - void close_closesRegularAndExpressClients() throws IOException { + void close_closesRegularAndAllRegionalExpressClients() throws IOException { S3Client regularClient = Mockito.mock(S3Client.class); - S3Client expressClient = Mockito.mock(S3Client.class); - S3ObjStorage expressStorage = directoryBucketStorage( - "https://s3.us-west-2.amazonaws.com", "us-west-2", false, - regularClient, expressClient); - Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + S3Client westExpressClient = Mockito.mock(S3Client.class); + S3Client eastExpressClient = Mockito.mock(S3Client.class); + Map props = directoryBucketProperties("", "", false); + props.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); + S3ObjStorage expressStorage = new TestableS3ObjStorage( + props, regularClient, region -> region.equals("us-west-2") + ? westExpressClient : eastExpressClient); + Mockito.when(westExpressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); + Mockito.when(eastExpressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) .thenReturn(ListObjectsV2Response.builder().contents(List.of()).isTruncated(false).build()); expressStorage.getClient(); - expressStorage.listObjects("s3://" + DIRECTORY_BUCKET + "/data/", null); + expressStorage.listObjects("s3://analytics--usw2-az1--x-s3/data/", null); + expressStorage.listObjects("s3://analytics--use1-az1--x-s3/data/", null); expressStorage.close(); Mockito.verify(regularClient).close(); - Mockito.verify(expressClient).close(); + Mockito.verify(westExpressClient).close(); + Mockito.verify(eastExpressClient).close(); } // ------------------------------------------------------------------ @@ -685,7 +792,8 @@ void close_closesRegularAndExpressClients() throws IOException { private static class TestableS3ObjStorage extends S3ObjStorage { private final S3Client mockClient; - private final S3Client mockExpressClient; + private final Function expressClientFactory; + private final List builtExpressRegions = new ArrayList<>(); TestableS3ObjStorage(Map properties, S3Client mockClient) { this(properties, mockClient, mockClient); @@ -693,15 +801,20 @@ private static class TestableS3ObjStorage extends S3ObjStorage { TestableS3ObjStorage(Map properties, S3Client mockClient, S3Client mockExpressClient) { + this(properties, mockClient, region -> mockExpressClient); + } + + TestableS3ObjStorage(Map properties, S3Client mockClient, + Function expressClientFactory) { super(properties); this.mockClient = mockClient; - this.mockExpressClient = mockExpressClient; + this.expressClientFactory = expressClientFactory; } TestableS3ObjStorage(S3FileSystemProperties properties, S3Client mockClient) { super(properties); this.mockClient = mockClient; - this.mockExpressClient = mockClient; + this.expressClientFactory = region -> mockClient; } @Override @@ -710,12 +823,17 @@ protected S3Client buildClient() { } @Override - protected S3Client buildExpressClient() { - return mockExpressClient; + protected S3Client buildExpressClient(String region) { + builtExpressRegions.add(region); + return expressClientFactory.apply(region); + } + + List getBuiltExpressRegions() { + return builtExpressRegions; } } - private static S3ObjStorage directoryBucketStorage(String endpoint, String region, + private static TestableS3ObjStorage directoryBucketStorage(String endpoint, String region, boolean usePathStyle, S3Client regularClient, S3Client expressClient) { Map props = directoryBucketProperties(endpoint, region, usePathStyle); props.put(S3FileSystemProperties.S3_EXPRESS_IMPORT_READ, "true"); @@ -727,6 +845,7 @@ private static Map directoryBucketProperties(String endpoint, St Map props = new HashMap<>(); props.put("AWS_ENDPOINT", endpoint); props.put("AWS_REGION", region); + props.put("provider", "AWS"); props.put("AWS_ACCESS_KEY", "testAK"); props.put("AWS_SECRET_KEY", "testSK"); props.put("AWS_BUCKET", DIRECTORY_BUCKET); @@ -735,6 +854,8 @@ private static Map directoryBucketProperties(String endpoint, St } private static class InspectableS3ObjStorage extends TestableS3ObjStorage { + private final List builtStsRegions = new ArrayList<>(); + InspectableS3ObjStorage(Map properties, S3Client mockClient) { super(properties, mockClient); } @@ -747,8 +868,17 @@ AwsCredentialsProvider inspectBuildCredentialsProvider() { return buildCredentialsProvider(); } + AwsCredentialsProvider inspectBuildCredentialsProvider(String region) { + return buildCredentialsProvider(region); + } + + List getBuiltStsRegions() { + return builtStsRegions; + } + @Override protected StsClient buildStsClient(AwsCredentialsProvider credentialsProvider, String region) { + builtStsRegions.add(region); return Mockito.mock(StsClient.class); } } diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageTest.java index c5c92619af314e..3e2ff057f00b3d 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageTest.java @@ -21,7 +21,9 @@ import org.junit.jupiter.api.Test; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetUrlRequest; +import java.net.URL; import java.util.HashMap; import java.util.Map; @@ -48,6 +50,30 @@ void getClient_endpointOnlyConfigurationUsesRegionBuiltByProperties() throws Exc } } + @Test + void buildExpressClient_usesSdkZonalEndpointAndVirtualHostedAddressing() throws Exception { + Map props = new HashMap<>(); + props.put("AWS_ENDPOINT", "https://endpoint-is-ignored.example.com"); + props.put("AWS_REGION", "us-east-1"); + props.put("AWS_ACCESS_KEY", "ak"); + props.put("AWS_SECRET_KEY", "sk"); + props.put("use_path_style", "true"); + + S3ObjStorage storage = new S3ObjStorage(props); + try (S3Client client = storage.buildExpressClient("us-west-2")) { + URL url = client.utilities().getUrl(GetUrlRequest.builder() + .bucket("analytics--usw2-az1--x-s3") + .key("data.parquet") + .build()); + + Assertions.assertEquals(Region.US_WEST_2, client.serviceClientConfiguration().region()); + Assertions.assertEquals("https", url.getProtocol()); + Assertions.assertEquals( + "analytics--usw2-az1--x-s3.s3express-usw2-az1.us-west-2.amazonaws.com", + url.getHost()); + } + } + // ------------------------------------------------------------------ // close() // ------------------------------------------------------------------ From 3f6d37f8a6a8a0fa0262a0ecacd5f4834804a7b8 Mon Sep 17 00:00:00 2001 From: Refrain Date: Sun, 19 Jul 2026 18:49:57 +0800 Subject: [PATCH 11/13] [fix](s3) Enable S3 Express reads for direct S3 TVF queries ### What problem does this PR solve? Issue Number: None Related PR: #63409 Problem Summary: The S3 Express statement-scope gate only admitted INSERT INTO SELECT and excluded a direct SELECT from the same S3 TVF, even though both statements share the same FE listing and BE read path. The gate also called Command.getType() for a real InsertIntoTableCommand, but Command does not implement that method. Recognize only the exact one-shot InsertIntoTableCommand and the parser's UnboundResultSink for a direct SELECT. This keeps CTAS, INSERT OVERWRITE, OUTFILE, COPY, FILE TVF, Streaming, internal commands, and specialized INSERT subclasses outside the trusted S3 Express marker boundary. ### Release note Support reading an Amazon S3 Express One Zone directory bucket with a direct SELECT from S3(...) in addition to INSERT INTO ... SELECT ... FROM S3(...) and Broker Load WITH S3. ### Check List (For Author) - Test: Unit Test - ./run-fe-ut.sh --run org.apache.doris.common.util.S3URITest,org.apache.doris.datasource.property.storage.S3PropertiesTest,org.apache.doris.fs.FileSystemFactoryS3ExpressScopeTest,org.apache.doris.load.loadv2.BrokerLoadJobTest,org.apache.doris.nereids.trees.plans.commands.LoadCommandTest,org.apache.doris.tablefunction.ExternalFileTableValuedFunctionTest,org.apache.doris.qe.S3ExpressImportScopeTest (58 tests passed) - Behavior changed: Yes. Direct one-shot SELECT from the S3 TVF can select the scoped S3 Express read client; non-target entry points remain excluded. - Does this need documentation: Yes. The PR description documents the supported entry points and configuration contract. --- .../doris/nereids/StatementContext.java | 2 +- .../org/apache/doris/qe/StmtExecutor.java | 6 +- .../doris/qe/S3ExpressImportScopeTest.java | 79 +++++++++++++------ .../ExternalFileTableValuedFunctionTest.java | 44 ++++++++--- 4 files changed, 93 insertions(+), 38 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 7e4e134584b883..b3a9abde76cd56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -314,7 +314,7 @@ public enum TableFrom { private final Set> materializationRewrittenSuccessSet = new HashSet<>(); private boolean isInsert = false; - // Trusted statement-scope marker for one-shot INSERT SELECT from the internal S3 TVF. + // Trusted statement-scope marker for one-shot SELECT/INSERT reads from the internal S3 TVF. private boolean s3ExpressImportRead = false; private Optional>> mvRefreshPredicates = Optional.empty(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index edf3c794a68bf5..c15a5a99ce7dac 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -81,6 +81,7 @@ import org.apache.doris.nereids.PlanProcess; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundBaseExternalTableSink; +import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.glue.LogicalPlanAdapter; @@ -91,7 +92,6 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner; import org.apache.doris.nereids.trees.plans.algebra.InlineTable; import org.apache.doris.nereids.trees.plans.commands.Command; @@ -987,8 +987,8 @@ static boolean shouldEnableS3ExpressImportRead(LogicalPlan logicalPlan, boolean if (logicalPlan instanceof ExplainCommand) { logicalPlan = ((ExplainCommand) logicalPlan).getLogicalPlan(); } - return logicalPlan instanceof InsertIntoTableCommand - && logicalPlan.getType() == PlanType.INSERT_INTO_TABLE_COMMAND; + return logicalPlan.getClass() == InsertIntoTableCommand.class + || logicalPlan instanceof UnboundResultSink; } public static void initBlockSqlAstNames() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java index 7147b65e0e3ba0..2b668d772d2f0a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/S3ExpressImportScopeTest.java @@ -17,41 +17,72 @@ package org.apache.doris.qe; +import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; -import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoDictionaryCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; -import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand; +import org.apache.doris.nereids.trees.plans.commands.insert.WarmupSelectCommand; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.util.Optional; - class S3ExpressImportScopeTest { + private static final String S3_SELECT_SQL = "SELECT * FROM S3(" + + "\"uri\" = \"s3://analytics--usw2-az1--x-s3/data/file.csv\", " + + "\"s3.provider\" = \"AWS\", " + + "\"format\" = \"csv\")"; + @Test - void enableOnlyForTopLevelOneShotInsertIntoTable() { - InsertIntoTableCommand insert = Mockito.mock(InsertIntoTableCommand.class); - Mockito.when(insert.getType()).thenReturn(PlanType.INSERT_INTO_TABLE_COMMAND); - - Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead(insert, false)); - Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead( - new ExplainCommand(ExplainLevel.NORMAL, insert, false), false)); - - Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead(insert, true)); - InsertOverwriteTableCommand overwrite = new InsertOverwriteTableCommand( - plan(PlanType.LOGICAL_RESULT_SINK), Optional.empty(), Optional.empty(), Optional.empty()); - Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( - overwrite, false)); - Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( - new ExplainCommand(ExplainLevel.NORMAL, overwrite, false), false)); - Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( - plan(PlanType.INSERT_INTO_TVF_COMMAND), false)); - Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( - plan(PlanType.LOGICAL_RESULT_SINK), false)); + void enableOnlyForTopLevelOneShotSelectAndInsert() { + ConnectContext previousContext = ConnectContext.get(); + ConnectContext context = new ConnectContext(); + context.setDatabase("test"); + context.setThreadLocalInfo(); + try { + NereidsParser parser = new NereidsParser(); + + LogicalPlan select = parser.parseSingle(S3_SELECT_SQL); + Assertions.assertEquals(PlanType.LOGICAL_UNBOUND_RESULT_SINK, select.getType()); + Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead(select, false)); + Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead( + parser.parseSingle("EXPLAIN " + S3_SELECT_SQL), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead(select, true)); + + LogicalPlan insert = parser.parseSingle("INSERT INTO target_table " + S3_SELECT_SQL); + Assertions.assertEquals(InsertIntoTableCommand.class, insert.getClass()); + Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead(insert, false)); + Assertions.assertTrue(StmtExecutor.shouldEnableS3ExpressImportRead( + parser.parseSingle("EXPLAIN INSERT INTO target_table " + S3_SELECT_SQL), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead(insert, true)); + + LogicalPlan overwrite = parser.parseSingle("INSERT OVERWRITE TABLE target_table " + S3_SELECT_SQL); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + overwrite, false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + parser.parseSingle("EXPLAIN INSERT OVERWRITE TABLE target_table " + S3_SELECT_SQL), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + parser.parseSingle("CREATE TABLE target_table AS " + S3_SELECT_SQL), false)); + LogicalPlan outfile = parser.parseSingle(S3_SELECT_SQL + + " INTO OUTFILE \"file:///tmp/s3-express-select\" FORMAT AS CSV"); + Assertions.assertEquals(PlanType.LOGICAL_FILE_SINK, outfile.getType()); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead(outfile, false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + plan(PlanType.INSERT_INTO_TVF_COMMAND), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + plan(PlanType.COPY_INTO_COMMAND), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + Mockito.mock(WarmupSelectCommand.class), false)); + Assertions.assertFalse(StmtExecutor.shouldEnableS3ExpressImportRead( + Mockito.mock(InsertIntoDictionaryCommand.class), false)); + } finally { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } } private static LogicalPlan plan(PlanType type) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index 37b726aa8eb6cc..79e9d6727f03b0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.expressions.Properties; +import org.apache.doris.nereids.trees.expressions.functions.table.File; import org.apache.doris.nereids.trees.expressions.functions.table.S3; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.OriginStatement; @@ -125,7 +126,7 @@ public void testCsvSchemaParse() { } @Test - public void testS3ExpressMarkerIsScopedToOneShotInsertStatement() throws AnalysisException { + public void testS3ExpressMarkerIsScopedToAllowedOneShotRead() throws AnalysisException { boolean previousRunningUnitTest = FeConstants.runningUnitTest; ConnectContext previousContext = ConnectContext.get(); FeConstants.runningUnitTest = true; @@ -145,30 +146,53 @@ public void testS3ExpressMarkerIsScopedToOneShotInsertStatement() throws Analysi ConnectContext context = new ConnectContext(); StatementContext statementContext = new StatementContext( - context, new OriginStatement("insert into t select * from s3(...)", 0)); + context, new OriginStatement("select * from s3(...)", 0)); context.setStatementContext(statementContext); context.setThreadLocalInfo(); - S3TableValuedFunction queryTvf = (S3TableValuedFunction) new S3(new Properties(properties)) + S3TableValuedFunction unmarkedTvf = (S3TableValuedFunction) new S3(new Properties(properties)) .getCatalogFunction(); - Assert.assertFalse(queryTvf.getBackendConnectProperties() + Assert.assertFalse(unmarkedTvf.getBackendConnectProperties() .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); statementContext.setS3ExpressImportRead(true); + Map regularBucketProperties = Maps.newHashMap(properties); + regularBucketProperties.put("uri", "s3://analytics/data/file.csv"); + S3TableValuedFunction regularBucketTvf = (S3TableValuedFunction) new S3( + new Properties(regularBucketProperties)).getCatalogFunction(); + Assert.assertFalse(regularBucketTvf.getBackendConnectProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertFalse(regularBucketTvf.getBrokerDesc().getBackendConfigProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + + Map missingProviderProperties = Maps.newHashMap(properties); + missingProviderProperties.remove("s3.provider"); + S3TableValuedFunction missingProviderTvf = (S3TableValuedFunction) new S3( + new Properties(missingProviderProperties)).getCatalogFunction(); + Assert.assertFalse(missingProviderTvf.getBackendConnectProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertFalse(missingProviderTvf.getBrokerDesc().getBackendConfigProperties() + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Map recommendedProperties = Maps.newHashMap(properties); recommendedProperties.remove("s3.endpoint"); recommendedProperties.remove("s3.region"); - S3TableValuedFunction insertTvf = (S3TableValuedFunction) new S3( + S3TableValuedFunction selectTvf = (S3TableValuedFunction) new S3( new Properties(recommendedProperties)) .getCatalogFunction(); - Assert.assertEquals("true", insertTvf.getBackendConnectProperties() + Assert.assertEquals("true", selectTvf.getBackendConnectProperties() .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); - Assert.assertEquals("true", insertTvf.getBrokerDesc().getBackendConfigProperties() + Assert.assertEquals("true", selectTvf.getBrokerDesc().getBackendConfigProperties() .get(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); - Assert.assertFalse(insertTvf.processedParams + Assert.assertFalse(selectTvf.processedParams + .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + Assert.assertEquals("", selectTvf.getBackendConnectProperties().get("AWS_ENDPOINT")); + Assert.assertEquals("", selectTvf.getBackendConnectProperties().get("AWS_REGION")); + + FileTableValuedFunction fileTvf = (FileTableValuedFunction) new File(new Properties(properties)) + .getCatalogFunction(); + Assert.assertFalse(fileTvf.getBackendConnectProperties() .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); - Assert.assertEquals("", insertTvf.getBackendConnectProperties().get("AWS_ENDPOINT")); - Assert.assertEquals("", insertTvf.getBackendConnectProperties().get("AWS_REGION")); statementContext.setS3ExpressImportRead(false); S3TableValuedFunction streamingTvf = (S3TableValuedFunction) new S3(new Properties(properties)) From 9195f7b601c72581f0f22c68ae9b5630d43980c2 Mon Sep 17 00:00:00 2001 From: Refrain Date: Sun, 19 Jul 2026 20:13:08 +0800 Subject: [PATCH 12/13] [fix](s3) Improve S3 Express import diagnostics ### What problem does this PR solve? Issue Number: None Related PR: #65504 Problem Summary: S3 Express import requests whose bucket clearly uses the --x-s3 suffix could silently fall back to generic S3-compatible provider detection when the statement scope, provider, or directory bucket format was invalid. This produced misleading MinIO or ordinary S3 errors. FE list failures also lost AWS service details and incorrectly implied a specific CreateSession or ListObjectsV2 stage. Recognize S3 Express intent before fallback, validate only the supported import entry points, and preserve HTTP status, AWS error code, message, and request ID with joint operation context. ### Release note S3 Express import requests now report clear scope, provider, bucket-format, and FE service errors instead of silently falling back to ordinary S3-compatible handling. ### Check List (For Author) - Test: Unit Test - FE unit tests for S3 Express intent validation, TVF and Broker Load scope, and S3 filesystem error reporting - Behavior changed: Yes. Invalid S3 Express import requests now fail with feature-specific diagnostics. - Does this need documentation: No --- .../property/storage/S3Properties.java | 86 +++++++++++++++++-- .../nereids/parser/LogicalPlanBuilder.java | 28 ++++-- .../FileTableValuedFunction.java | 2 + .../tablefunction/S3TableValuedFunction.java | 6 +- .../property/storage/S3PropertiesTest.java | 53 ++++++++++++ .../trees/plans/commands/LoadCommandTest.java | 23 +++++ .../ExternalFileTableValuedFunctionTest.java | 56 +++++++----- .../doris/filesystem/s3/S3ObjStorage.java | 17 ++++ .../filesystem/s3/S3ObjStorageMockTest.java | 81 +++++++++++++++++ 9 files changed, 312 insertions(+), 40 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java index 36bcdfe53f2bde..b5b3613ff263f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java @@ -58,6 +58,9 @@ public class S3Properties extends AbstractS3CompatibleProperties { private static final Logger LOG = LogManager.getLogger(S3Properties.class); + private static final String S3_DIRECTORY_BUCKET_SUFFIX = "--x-s3"; + private static final Pattern S3_EXPRESS_URI_AUTHORITY_PATTERN = + Pattern.compile("^[^:/?#]+://([^/?#]+)"); public static final String USE_PATH_STYLE = "use_path_style"; public static final String ENDPOINT = "s3.endpoint"; @@ -215,11 +218,11 @@ public static S3Properties createForS3ExpressImport(Map properti /** Returns whether the user explicitly selected AWS as the S3 provider. */ public static boolean isAwsProvider(Map properties) { - return properties.entrySet().stream() - .filter(entry -> entry.getKey().equalsIgnoreCase(PROVIDER) - || entry.getKey().equalsIgnoreCase("provider")) - .map(Map.Entry::getValue) - .anyMatch(value -> "AWS".equalsIgnoreCase(value)); + Optional provider = getPropertyIgnoreCase(properties, PROVIDER); + if (provider.isEmpty()) { + provider = getPropertyIgnoreCase(properties, "provider"); + } + return provider.map(String::trim).filter(value -> "AWS".equalsIgnoreCase(value)).isPresent(); } /** Returns whether a scoped S3 TVF request can use SDK-managed S3 Express access. */ @@ -227,16 +230,81 @@ public static boolean isS3ExpressImport(Map properties) { if (!isAwsProvider(properties)) { return false; } - Optional uri = properties.entrySet().stream() - .filter(entry -> entry.getKey().equalsIgnoreCase("uri")) - .map(Map.Entry::getValue) - .findFirst(); + Optional uri = getPropertyIgnoreCase(properties, "uri"); if (uri.isEmpty()) { return false; } return isS3ExpressUri(uri.get()); } + /** + * Validates an S3 Express request after first recognizing the user's intent from the bucket suffix. + * Ordinary S3 URIs return false without changing their existing provider detection path. + */ + public static boolean validateS3ExpressImport(Map properties, boolean scopeAllowed) { + return getPropertyIgnoreCase(properties, "uri") + .map(uri -> validateS3ExpressImport(uri, properties, scopeAllowed)) + .orElse(false); + } + + /** + * Validates an S3 Express request whose URI is supplied separately, as in Broker Load. + */ + public static boolean validateS3ExpressImport( + String uri, Map properties, boolean scopeAllowed) { + if (!hasS3ExpressIntent(uri)) { + return false; + } + if (!scopeAllowed) { + throw new IllegalArgumentException("S3 Express reads are supported only by " + + "SELECT ... FROM S3(...), INSERT INTO ... SELECT ... FROM S3(...), " + + "and Broker Load WITH S3."); + } + if (!isAwsProvider(properties)) { + throw new IllegalArgumentException( + "S3 Express directory buckets require \"s3.provider\" = \"AWS\"."); + } + + S3URI parsedUri; + try { + parsedUri = S3URI.create(uri); + } catch (UserException e) { + throw new IllegalArgumentException("Invalid S3 Express URI \"" + uri + "\"; expected " + + "\"s3://----x-s3/\".", e); + } + if (!parsedUri.useS3DirectoryBucket()) { + throw new IllegalArgumentException("Invalid S3 Express directory bucket name \"" + + parsedUri.getBucket() + "\"; expected " + + "\"----x-s3\", for example " + + "\"bucket-name--usw2-az1--x-s3\"."); + } + return true; + } + + /** Returns whether the URI bucket clearly carries the S3 Express suffix. */ + public static boolean hasS3ExpressIntent(String uri) { + if (StringUtils.isBlank(uri)) { + return false; + } + try { + return hasS3ExpressBucketSuffix(S3URI.create(uri).getBucket()); + } catch (UserException e) { + java.util.regex.Matcher matcher = S3_EXPRESS_URI_AUTHORITY_PATTERN.matcher(uri); + return matcher.find() && hasS3ExpressBucketSuffix(matcher.group(1)); + } + } + + private static boolean hasS3ExpressBucketSuffix(String bucket) { + return StringUtils.endsWithIgnoreCase(bucket, S3_DIRECTORY_BUCKET_SUFFIX); + } + + private static Optional getPropertyIgnoreCase(Map properties, String propertyName) { + return properties.entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(propertyName)) + .map(Map.Entry::getValue) + .findFirst(); + } + /** Returns whether the URI contains a complete S3 Express directory bucket name. */ public static boolean isS3ExpressUri(String uri) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index b9186781ef458c..59a46c5532ede5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -2231,13 +2231,31 @@ public List> visitMultiStatements(MultiState */ @Override public LogicalPlan visitLoad(DorisParser.LoadContext ctx) { + List allFilePaths = ctx.dataDescs.stream() + .flatMap(dataDesc -> dataDesc.filePaths.stream()) + .map(Token::getText) + .map(path -> path.substring(1, path.length() - 1)) + .collect(ImmutableList.toImmutableList()); + boolean hasS3ExpressIntent = allFilePaths.stream().anyMatch(S3Properties::hasS3ExpressIntent); + if (hasS3ExpressIntent && (ctx.withRemoteStorageSystem() == null + || ctx.withRemoteStorageSystem().S3() == null)) { + allFilePaths.stream() + .filter(S3Properties::hasS3ExpressIntent) + .findFirst() + .ifPresent(path -> S3Properties.validateS3ExpressImport(path, ImmutableMap.of(), false)); + } + BrokerDesc brokerDesc = null; if (ctx.withRemoteStorageSystem() != null) { - boolean s3ExpressImportRead = ctx.dataDescs.stream() - .flatMap(dataDesc -> dataDesc.filePaths.stream()) - .map(Token::getText) - .map(path -> path.substring(1, path.length() - 1)) - .anyMatch(S3Properties::isS3ExpressUri); + boolean s3ExpressImportRead = false; + if (hasS3ExpressIntent) { + Map brokerProperties = + visitPropertyItemList(ctx.withRemoteStorageSystem().brokerProperties); + for (String filePath : allFilePaths) { + s3ExpressImportRead |= S3Properties.validateS3ExpressImport( + filePath, brokerProperties, true); + } + } brokerDesc = visitWithRemoteStorageSystem( ctx.withRemoteStorageSystem(), s3ExpressImportRead); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FileTableValuedFunction.java index 2cb5bedb4463ce..b14397813a448f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/FileTableValuedFunction.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.HttpProperties; import org.apache.doris.datasource.property.storage.LocalProperties; +import org.apache.doris.datasource.property.storage.S3Properties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.ScanNode; @@ -50,6 +51,7 @@ public class FileTableValuedFunction extends ExternalFileTableValuedFunction { public FileTableValuedFunction(Map properties) throws AnalysisException { // We don't need to parseCommonProperties because the corresponding Storage will do it // Map props = super.parseCommonProperties(properties); + S3Properties.validateS3ExpressImport(properties, false); try { this.storageProperties = StorageProperties.createPrimary(properties); if (this.storageProperties instanceof AbstractS3CompatibleProperties diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 35f1dca5a3bdf0..f7d2a48300c5e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -48,11 +48,11 @@ public S3TableValuedFunction(Map properties) throws AnalysisExce public S3TableValuedFunction(Map properties, boolean s3ExpressImportRead) throws AnalysisException { - this.s3ExpressImportRead = s3ExpressImportRead; // 1. analyze common properties Map props = super.parseCommonProperties(properties); + this.s3ExpressImportRead = S3Properties.validateS3ExpressImport(props, s3ExpressImportRead); try { - if (s3ExpressImportRead && S3Properties.isS3ExpressImport(props)) { + if (this.s3ExpressImportRead) { this.storageProperties = S3Properties.createForS3ExpressImport(props); } else { this.storageProperties = StorageProperties.createPrimary(props); @@ -88,7 +88,7 @@ public String getFilePath() { @Override public BrokerDesc getBrokerDesc() { - if (s3ExpressImportRead && S3Properties.isS3ExpressImport(processedParams)) { + if (s3ExpressImportRead) { return BrokerDesc.createForS3ExpressImport("S3TvfBroker", processedParams); } return new BrokerDesc("S3TvfBroker", processedParams); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java index be3b6022b3039f..6398d0eaf148bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java @@ -560,4 +560,57 @@ public void testS3ExpressImportAllowsSdkEndpointResolution() { Assertions.assertEquals("", backendProperties.get("AWS_ENDPOINT")); Assertions.assertEquals("", backendProperties.get("AWS_REGION")); } + + @Test + public void testS3ExpressIntentValidation() { + origProps.put("uri", "s3://analytics--usw2-az1--x-s3/data/file.parquet"); + origProps.put("s3.provider", " AWS "); + + Assertions.assertTrue(S3Properties.hasS3ExpressIntent(origProps.get("uri"))); + Assertions.assertTrue(S3Properties.validateS3ExpressImport(origProps, true)); + Assertions.assertFalse(S3Properties.hasS3ExpressIntent( + "s3://ordinary-bucket/data/part--x-s3/file.parquet")); + Assertions.assertFalse(S3Properties.validateS3ExpressImport( + "s3://ordinary-bucket/data/file.parquet", origProps, true)); + + IllegalArgumentException scopeException = Assertions.assertThrows(IllegalArgumentException.class, + () -> S3Properties.validateS3ExpressImport(origProps, false)); + Assertions.assertTrue(scopeException.getMessage().contains( + "SELECT ... FROM S3(...), INSERT INTO ... SELECT ... FROM S3(...), and Broker Load WITH S3")); + + Map missingProvider = new HashMap<>(origProps); + missingProvider.remove("s3.provider"); + IllegalArgumentException providerException = Assertions.assertThrows(IllegalArgumentException.class, + () -> S3Properties.validateS3ExpressImport(missingProvider, true)); + Assertions.assertEquals( + "S3 Express directory buckets require \"s3.provider\" = \"AWS\".", + providerException.getMessage()); + + Map invalidBucket = new HashMap<>(origProps); + invalidBucket.put("uri", "s3://analytics--usw2-azx--x-s3/data/file.parquet"); + Assertions.assertTrue(S3Properties.hasS3ExpressIntent(invalidBucket.get("uri"))); + IllegalArgumentException bucketException = Assertions.assertThrows(IllegalArgumentException.class, + () -> S3Properties.validateS3ExpressImport(invalidBucket, true)); + Assertions.assertTrue(bucketException.getMessage().contains( + "Invalid S3 Express directory bucket name \"analytics--usw2-azx--x-s3\"")); + + Map missingObjectPath = new HashMap<>(origProps); + missingObjectPath.put("uri", "s3://analytics--usw2-az1--x-s3"); + Assertions.assertTrue(S3Properties.hasS3ExpressIntent(missingObjectPath.get("uri"))); + IllegalArgumentException uriException = Assertions.assertThrows(IllegalArgumentException.class, + () -> S3Properties.validateS3ExpressImport(missingObjectPath, true)); + Assertions.assertTrue(uriException.getMessage().contains("Invalid S3 Express URI")); + Assertions.assertTrue(uriException.getMessage().contains("/")); + } + + @Test + public void testS3ExpressProviderUsesCanonicalAliasPrecedence() { + origProps.put("s3.provider", "S3"); + origProps.put("provider", "AWS"); + Assertions.assertFalse(S3Properties.isAwsProvider(origProps)); + + origProps.put("s3.provider", "AWS"); + origProps.put("provider", "S3"); + Assertions.assertTrue(S3Properties.isAwsProvider(origProps)); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java index b1ae621b0632ec..8caa01abe9ef05 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java @@ -169,6 +169,29 @@ public void testS3ExpressLoadWithoutEndpointAndRegion() { Assertions.assertEquals("", backendProperties.get("AWS_ENDPOINT")); Assertions.assertEquals("", backendProperties.get("AWS_REGION")); + String missingProviderSql = loadSql.replace( + " \"s3.provider\" = \"AWS\", ", ""); + IllegalArgumentException providerException = Assertions.assertThrows( + IllegalArgumentException.class, + () -> new NereidsParser().parseMultiple(missingProviderSql)); + Assertions.assertTrue(providerException.getMessage().contains( + "S3 Express directory buckets require \"s3.provider\" = \"AWS\"")); + + String malformedBucketSql = loadSql.replace( + "analytics--usw2-az1--x-s3", "analytics--usw2-azx--x-s3"); + IllegalArgumentException bucketException = Assertions.assertThrows( + IllegalArgumentException.class, + () -> new NereidsParser().parseMultiple(malformedBucketSql)); + Assertions.assertTrue(bucketException.getMessage().contains( + "Invalid S3 Express directory bucket name")); + + String brokerScopeSql = loadSql.replace("WITH S3(", "WITH BROKER \"broker0\" ("); + IllegalArgumentException scopeException = Assertions.assertThrows( + IllegalArgumentException.class, + () -> new NereidsParser().parseMultiple(brokerScopeSql)); + Assertions.assertTrue(scopeException.getMessage().contains( + "S3 Express reads are supported only by")); + String ordinaryBucketSql = loadSql.replace( "analytics--usw2-az1--x-s3", "ordinary-bucket"); Assertions.assertThrows(IllegalArgumentException.class, diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index 79e9d6727f03b0..7ed802f20bf68a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -138,11 +138,10 @@ public void testS3ExpressMarkerIsScopedToAllowedOneShotRead() throws AnalysisExc properties.put("s3.region", "us-west-2"); properties.put("format", "csv"); - S3TableValuedFunction directTvf = new S3TableValuedFunction(properties); - Assert.assertFalse(directTvf.getBackendConnectProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); - Assert.assertFalse(directTvf.getBrokerDesc().getBackendConfigProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + IllegalArgumentException directException = Assert.assertThrows( + IllegalArgumentException.class, () -> new S3TableValuedFunction(properties)); + Assert.assertTrue(directException.getMessage().contains( + "S3 Express reads are supported only by")); ConnectContext context = new ConnectContext(); StatementContext statementContext = new StatementContext( @@ -150,10 +149,11 @@ public void testS3ExpressMarkerIsScopedToAllowedOneShotRead() throws AnalysisExc context.setStatementContext(statementContext); context.setThreadLocalInfo(); - S3TableValuedFunction unmarkedTvf = (S3TableValuedFunction) new S3(new Properties(properties)) - .getCatalogFunction(); - Assert.assertFalse(unmarkedTvf.getBackendConnectProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + org.apache.doris.nereids.exceptions.AnalysisException unmarkedException = Assert.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new S3(new Properties(properties)).getCatalogFunction()); + Assert.assertTrue(unmarkedException.getMessage().contains( + "S3 Express reads are supported only by")); statementContext.setS3ExpressImportRead(true); Map regularBucketProperties = Maps.newHashMap(properties); @@ -167,12 +167,20 @@ public void testS3ExpressMarkerIsScopedToAllowedOneShotRead() throws AnalysisExc Map missingProviderProperties = Maps.newHashMap(properties); missingProviderProperties.remove("s3.provider"); - S3TableValuedFunction missingProviderTvf = (S3TableValuedFunction) new S3( - new Properties(missingProviderProperties)).getCatalogFunction(); - Assert.assertFalse(missingProviderTvf.getBackendConnectProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); - Assert.assertFalse(missingProviderTvf.getBrokerDesc().getBackendConfigProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + org.apache.doris.nereids.exceptions.AnalysisException missingProviderException = Assert.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new S3(new Properties(missingProviderProperties)).getCatalogFunction()); + Assert.assertTrue(missingProviderException.getMessage().contains( + "S3 Express directory buckets require \"s3.provider\" = \"AWS\"")); + + Map malformedBucketProperties = Maps.newHashMap(properties); + malformedBucketProperties.put("uri", + "s3://analytics--usw2-azx--x-s3/data/file.csv"); + org.apache.doris.nereids.exceptions.AnalysisException malformedBucketException = Assert.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new S3(new Properties(malformedBucketProperties)).getCatalogFunction()); + Assert.assertTrue(malformedBucketException.getMessage().contains( + "Invalid S3 Express directory bucket name")); Map recommendedProperties = Maps.newHashMap(properties); recommendedProperties.remove("s3.endpoint"); @@ -189,16 +197,18 @@ public void testS3ExpressMarkerIsScopedToAllowedOneShotRead() throws AnalysisExc Assert.assertEquals("", selectTvf.getBackendConnectProperties().get("AWS_ENDPOINT")); Assert.assertEquals("", selectTvf.getBackendConnectProperties().get("AWS_REGION")); - FileTableValuedFunction fileTvf = (FileTableValuedFunction) new File(new Properties(properties)) - .getCatalogFunction(); - Assert.assertFalse(fileTvf.getBackendConnectProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + org.apache.doris.nereids.exceptions.AnalysisException fileTvfException = Assert.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new File(new Properties(recommendedProperties)).getCatalogFunction()); + Assert.assertTrue(fileTvfException.getMessage().contains( + "S3 Express reads are supported only by")); statementContext.setS3ExpressImportRead(false); - S3TableValuedFunction streamingTvf = (S3TableValuedFunction) new S3(new Properties(properties)) - .getCatalogFunction(); - Assert.assertFalse(streamingTvf.getBackendConnectProperties() - .containsKey(AbstractS3CompatibleProperties.S3_EXPRESS_IMPORT_READ)); + org.apache.doris.nereids.exceptions.AnalysisException streamingException = Assert.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new S3(new Properties(properties)).getCatalogFunction()); + Assert.assertTrue(streamingException.getMessage().contains( + "S3 Express reads are supported only by")); } finally { FeConstants.runningUnitTest = previousRunningUnitTest; ConnectContext.remove(); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java index cf0b3acafa0f3c..ee6cdaae542251 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3ObjStorage.java @@ -296,11 +296,28 @@ public RemoteObjects listObjectsWithOptions(String remotePath, ObjectListOptions .collect(Collectors.toList()); return new RemoteObjects(objects, response.isTruncated(), response.nextContinuationToken()); + } catch (S3Exception e) { + if (expressRead) { + throw new IOException("Create S3 Express session or list directory bucket failed for " + + remotePath + ": " + formatS3Exception(e), e); + } + throw new IOException("Failed to list objects at " + remotePath + ": " + e.getMessage(), e); } catch (SdkException e) { + if (expressRead) { + throw new IOException("Create S3 Express session or list directory bucket failed for " + + remotePath + ": " + e.getMessage(), e); + } throw new IOException("Failed to list objects at " + remotePath + ": " + e.getMessage(), e); } } + private static String formatS3Exception(S3Exception exception) { + return "HTTP status=" + exception.statusCode() + + ", AWS error code=" + exception.awsErrorDetails().errorCode() + + ", message=" + exception.awsErrorDetails().errorMessage() + + ", request ID=" + exception.requestId(); + } + /** * Strips {@code prefix} (with implicit trailing-slash normalisation) from {@code key} * to produce the per-listing relative path. Identical normalisation to diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java index c8cf3a0e5d7610..cc6a74c26a937f 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3ObjStorageMockTest.java @@ -32,6 +32,8 @@ import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider; +import software.amazon.awssdk.awscore.exception.AwsErrorDetails; +import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; @@ -214,6 +216,85 @@ void listObjects_directoryBucketRootDoesNotSendPrefix() throws IOException { Mockito.verifyNoInteractions(regularClient); } + @Test + void listObjects_directoryBucketS3ExceptionPreservesServiceErrorDetails() { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "", "us-west-2", false, regularClient, expressClient); + S3Exception failure = (S3Exception) S3Exception.builder() + .statusCode(403) + .requestId("request-123") + .awsErrorDetails(AwsErrorDetails.builder() + .serviceName("S3") + .errorCode("AccessDenied") + .errorMessage("CreateSession is not allowed") + .build()) + .message("CreateSession is not allowed") + .build(); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenThrow(failure); + + IOException exception = Assertions.assertThrows(IOException.class, + () -> expressStorage.listObjects( + "s3://" + DIRECTORY_BUCKET + "/data/file.parquet", null)); + + Assertions.assertAll( + () -> Assertions.assertTrue(exception.getMessage().contains( + "Create S3 Express session or list directory bucket failed")), + () -> Assertions.assertTrue(exception.getMessage().contains("HTTP status=403")), + () -> Assertions.assertTrue(exception.getMessage().contains("AWS error code=AccessDenied")), + () -> Assertions.assertTrue(exception.getMessage().contains( + "message=CreateSession is not allowed")), + () -> Assertions.assertTrue(exception.getMessage().contains("request ID=request-123")), + () -> Assertions.assertSame(failure, exception.getCause())); + Mockito.verifyNoInteractions(regularClient); + } + + @Test + void listObjects_directoryBucketSdkExceptionUsesCombinedOperationContext() { + S3Client regularClient = Mockito.mock(S3Client.class); + S3Client expressClient = Mockito.mock(S3Client.class); + S3ObjStorage expressStorage = directoryBucketStorage( + "", "us-west-2", false, regularClient, expressClient); + SdkException failure = SdkException.builder().message("network down").build(); + Mockito.when(expressClient.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenThrow(failure); + + IOException exception = Assertions.assertThrows(IOException.class, + () -> expressStorage.listObjects( + "s3://" + DIRECTORY_BUCKET + "/data/file.parquet", null)); + + Assertions.assertTrue(exception.getMessage().contains( + "Create S3 Express session or list directory bucket failed")); + Assertions.assertTrue(exception.getMessage().contains("network down")); + Assertions.assertSame(failure, exception.getCause()); + Mockito.verifyNoInteractions(regularClient); + } + + @Test + void listObjects_regularBucketKeepsExistingS3ExceptionContext() { + S3Exception failure = (S3Exception) S3Exception.builder() + .statusCode(403) + .requestId("request-ordinary") + .awsErrorDetails(AwsErrorDetails.builder() + .serviceName("S3") + .errorCode("AccessDenied") + .errorMessage("Forbidden") + .build()) + .message("Forbidden") + .build(); + Mockito.when(mockS3.listObjectsV2(ArgumentMatchers.any(ListObjectsV2Request.class))) + .thenThrow(failure); + + IOException exception = Assertions.assertThrows(IOException.class, + () -> storage.listObjects("s3://my-bucket/data/file.parquet", null)); + + Assertions.assertTrue(exception.getMessage().startsWith("Failed to list objects at")); + Assertions.assertFalse(exception.getMessage().contains("S3 Express")); + Assertions.assertSame(failure, exception.getCause()); + } + @Test void listObjects_directoryBucketWithoutImportMarkerStaysOnRegularClient() throws IOException { S3Client regularClient = Mockito.mock(S3Client.class); From 276bd1e997e253e3cfa747dfc566105aeb6def3f Mon Sep 17 00:00:00 2001 From: Refrain Date: Sun, 19 Jul 2026 21:57:06 +0800 Subject: [PATCH 13/13] [fix](fe) Fix ASF header in S3 Express scope test ### What problem does this PR solve? Issue Number: None Related PR: #65504 Problem Summary: FileSystemFactoryS3ExpressScopeTest used nonstandard line wrapping in the ASF license header. FE Checkstyle rejected the header and blocked the Compile, FE UT, and Performance pipelines before fe-core compilation. Align the file with the repository standard ASF header. ### Release note None ### Check List (For Author) - Test: Unit Test - Debug FE build: ./build.sh --fe -j48 - FE UT: ./run-fe-ut.sh --run org.apache.doris.fs.FileSystemFactoryS3ExpressScopeTest - Behavior changed: No - Does this need documentation: No --- .../doris/fs/FileSystemFactoryS3ExpressScopeTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java index 7e8004385b4766..b0a88a8c98878c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemFactoryS3ExpressScopeTest.java @@ -8,11 +8,12 @@ // // http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. package org.apache.doris.fs;