From 9e2ce439349a36340a72f2d8b3c1fc952346873b Mon Sep 17 00:00:00 2001 From: jiuker Date: Wed, 12 Aug 2026 11:32:23 +0800 Subject: [PATCH 1/6] fix: Windows large file support and binary-safe downloads Fixes two Windows (LLP64) issues reported via the VCPKG port: binary file corruption on download and 32-bit 'long' overflow for files larger than 2 GiB. Binary-safe downloads: - DownloadObject: open the temp output stream with std::ios::binary. On Windows the default text mode translates newlines and treats 0x1A as EOF, corrupting downloaded binary content (e.g. zip archives). Large file support (LLP64: 'long' is strictly 32-bit on Windows): - SelectResult (types.h) and the parser in select.cc: bytes_scanned / bytes_processed / bytes_returned switch from 'long int' to 'long long', and std::stol to std::stoll - StatObject (baseclient.cc): parse the content-length header with std::stoll so resp.size (size_t) is not truncated above 2 GiB - ComposeSource (args.h/args.cc): object_size_ member and its assignment switch from 'long' to 'long long' so composing sources larger than 2 GiB does not overflow No behavioral change on LP64 platforms (Linux/macOS), where 'long' is already 64-bit. Files changed: include/miniocpp/args.h, include/miniocpp/types.h, src/args.cc, src/baseclient.cc, src/client.cc, src/select.cc --- include/miniocpp/args.h | 2 +- include/miniocpp/types.h | 10 +++++----- src/args.cc | 2 +- src/baseclient.cc | 2 +- src/client.cc | 3 ++- src/select.cc | 12 ++++++------ 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/include/miniocpp/args.h b/include/miniocpp/args.h index 19870928..9d4716c7 100644 --- a/include/miniocpp/args.h +++ b/include/miniocpp/args.h @@ -382,7 +382,7 @@ struct ComposeSource : public ObjectConditionalReadArgs { utils::Multimap Headers() const; private: - long object_size_ = -1; + long long object_size_ = -1; utils::Multimap headers_; }; // struct ComposeSource diff --git a/include/miniocpp/types.h b/include/miniocpp/types.h index 5be1ae42..8266c88f 100644 --- a/include/miniocpp/types.h +++ b/include/miniocpp/types.h @@ -293,17 +293,17 @@ struct SelectRequest { struct SelectResult { error::Error err = error::SUCCESS; bool ended = false; - long int bytes_scanned = -1; - long int bytes_processed = -1; - long int bytes_returned = -1; + long long bytes_scanned = -1; + long long bytes_processed = -1; + long long bytes_returned = -1; std::string records; SelectResult() : ended(true) {} explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {} - SelectResult(long int bytes_scanned, long int bytes_processed, - long int bytes_returned) + SelectResult(long long bytes_scanned, long long bytes_processed, + long long bytes_returned) : bytes_scanned(bytes_scanned), bytes_processed(bytes_processed), bytes_returned(bytes_returned) {} diff --git a/src/args.cc b/src/args.cc index deb8e022..66b463a0 100644 --- a/src/args.cc +++ b/src/args.cc @@ -381,7 +381,7 @@ error::Error ComposeSource::BuildHeaders(size_t object_size, } } - object_size_ = static_cast(object_size); + object_size_ = static_cast(object_size); headers_ = CopyHeaders(); if (!headers_.Contains("x-amz-copy-source-if-match")) { headers_.Add("x-amz-copy-source-if-match", etag); diff --git a/src/baseclient.cc b/src/baseclient.cc index 492f5014..1196cf46 100644 --- a/src/baseclient.cc +++ b/src/baseclient.cc @@ -2049,7 +2049,7 @@ Result BaseClient::StatObject(StatObjectArgs args) { resp.etag = utils::Trim(response->headers.GetFront("etag"), '"'); std::string value = response->headers.GetFront("content-length"); - if (!value.empty()) resp.size = std::stol(value); + if (!value.empty()) resp.size = std::stoll(value); value = response->headers.GetFront("last-modified"); if (!value.empty()) { diff --git a/src/client.cc b/src/client.cc index c1045d74..433305e3 100644 --- a/src/client.cc +++ b/src/client.cc @@ -1165,7 +1165,8 @@ Result Client::DownloadObject(DownloadObjectArgs args) { std::string temp_filename = args.filename + "." + curlpp::escape(etag) + ".part.minio"; - std::ofstream fout(temp_filename, std::ios::trunc | std::ios::out); + std::ofstream fout(temp_filename, + std::ios::trunc | std::ios::out | std::ios::binary); if (!fout.is_open()) { return error::make("unable to open file " + temp_filename); diff --git a/src/select.cc b/src/select.cc index ce6c4db4..8cdbea65 100644 --- a/src/select.cc +++ b/src/select.cc @@ -195,21 +195,21 @@ bool SelectHandler::process(const http::DataFunctionArgs& /* args */, auto root = xdoc.select_node(xpath.c_str()); pugi::xpath_node text; std::string value; - long int bytes_scanned = -1; - long int bytes_processed = -1; - long int bytes_returned = -1; + long long bytes_scanned = -1; + long long bytes_processed = -1; + long long bytes_returned = -1; text = root.node().select_node("BytesScanned/text()"); value = text.node().value(); - if (!value.empty()) bytes_scanned = std::stol(value); + if (!value.empty()) bytes_scanned = std::stoll(value); text = root.node().select_node("BytesProcessed/text()"); value = text.node().value(); - if (!value.empty()) bytes_processed = std::stol(value); + if (!value.empty()) bytes_processed = std::stoll(value); text = root.node().select_node("BytesReturned/text()"); value = text.node().value(); - if (!value.empty()) bytes_returned = std::stol(value); + if (!value.empty()) bytes_returned = std::stoll(value); cont = result_func_( SelectResult(bytes_scanned, bytes_processed, bytes_returned)); From a788fe326776f6a7573af2c4e3194fa818641808 Mon Sep 17 00:00:00 2001 From: jiuker Date: Thu, 13 Aug 2026 15:38:41 +0800 Subject: [PATCH 2/6] apply suggestion apply suggestion --- include/miniocpp/args.h | 2 +- include/miniocpp/types.h | 12 +++++++----- src/args.cc | 6 +++--- src/select.cc | 6 +++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/include/miniocpp/args.h b/include/miniocpp/args.h index 9d4716c7..ded4a4fb 100644 --- a/include/miniocpp/args.h +++ b/include/miniocpp/args.h @@ -382,7 +382,7 @@ struct ComposeSource : public ObjectConditionalReadArgs { utils::Multimap Headers() const; private: - long long object_size_ = -1; + std::optional object_size_; utils::Multimap headers_; }; // struct ComposeSource diff --git a/include/miniocpp/types.h b/include/miniocpp/types.h index 8266c88f..98e2910d 100644 --- a/include/miniocpp/types.h +++ b/include/miniocpp/types.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -293,17 +294,18 @@ struct SelectRequest { struct SelectResult { error::Error err = error::SUCCESS; bool ended = false; - long long bytes_scanned = -1; - long long bytes_processed = -1; - long long bytes_returned = -1; + std::optional bytes_scanned; + std::optional bytes_processed; + std::optional bytes_returned; std::string records; SelectResult() : ended(true) {} explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {} - SelectResult(long long bytes_scanned, long long bytes_processed, - long long bytes_returned) + SelectResult(std::optional bytes_scanned, + std::optional bytes_processed, + std::optional bytes_returned) : bytes_scanned(bytes_scanned), bytes_processed(bytes_processed), bytes_returned(bytes_returned) {} diff --git a/src/args.cc b/src/args.cc index 66b463a0..e4a89eb8 100644 --- a/src/args.cc +++ b/src/args.cc @@ -381,7 +381,7 @@ error::Error ComposeSource::BuildHeaders(size_t object_size, } } - object_size_ = static_cast(object_size); + object_size_ = object_size; headers_ = CopyHeaders(); if (!headers_.Contains("x-amz-copy-source-if-match")) { headers_.Add("x-amz-copy-source-if-match", etag); @@ -391,14 +391,14 @@ error::Error ComposeSource::BuildHeaders(size_t object_size, } size_t ComposeSource::ObjectSize() const { - if (object_size_ == -1) { + if (!object_size_.has_value()) { std::cerr << "ABORT: ComposeSource::BuildHeaders() must be called prior to " "this method invocation. This should not happen." << std::endl; std::terminate(); } - return object_size_; + return *object_size_; } utils::Multimap ComposeSource::Headers() const { diff --git a/src/select.cc b/src/select.cc index 8cdbea65..6851b35f 100644 --- a/src/select.cc +++ b/src/select.cc @@ -195,9 +195,9 @@ bool SelectHandler::process(const http::DataFunctionArgs& /* args */, auto root = xdoc.select_node(xpath.c_str()); pugi::xpath_node text; std::string value; - long long bytes_scanned = -1; - long long bytes_processed = -1; - long long bytes_returned = -1; + std::optional bytes_scanned; + std::optional bytes_processed; + std::optional bytes_returned; text = root.node().select_node("BytesScanned/text()"); value = text.node().value(); From 693c21c4d5c06357ebb730d4bc8b6e604273e6c3 Mon Sep 17 00:00:00 2001 From: jiuker Date: Thu, 13 Aug 2026 15:43:52 +0800 Subject: [PATCH 3/6] add test --- tests/tests.cc | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/tests.cc b/tests/tests.cc index 7d4de3b6..720b56e0 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include using minio::Result; @@ -38,6 +39,7 @@ using minio::Result; #include #include #include +#include #include #include #include @@ -102,6 +104,43 @@ std::string RandBucketName() { std::string RandObjectName() { return RandomString(charset, 8); } +std::string PutUint32BigEndian(unsigned int v) { + std::string s(4, '\0'); + s[0] = static_cast((v >> 24) & 0xFF); + s[1] = static_cast((v >> 16) & 0xFF); + s[2] = static_cast((v >> 8) & 0xFF); + s[3] = static_cast(v & 0xFF); + return s; +} + +// Build a single S3 Select protocol frame (prelude + prelude CRC + headers +// + payload + message CRC) for the given event headers and XML payload. +std::string MakeSelectFrame(const std::map& headers, + const std::string& payload) { + std::string headerdata; + for (const auto& [name, value] : headers) { + headerdata += static_cast(name.length()); + headerdata += name; + headerdata += static_cast(7); // header value type: string + headerdata += static_cast((value.length() >> 8) & 0xFF); + headerdata += static_cast(value.length() & 0xFF); + headerdata += value; + } + headerdata += static_cast(0); // header terminator + + std::string data = headerdata + payload; + unsigned int total_length = 16 + static_cast(data.length()); + std::string prelude = + PutUint32BigEndian(total_length) + + PutUint32BigEndian(static_cast(headerdata.length())); + std::string prelude_crc = + PutUint32BigEndian(static_cast(utils::CRC32(prelude))); + std::string message = prelude + prelude_crc + data; + std::string message_crc = + PutUint32BigEndian(static_cast(utils::CRC32(message))); + return message + message_crc; +} + struct MakeBucketError : public std::runtime_error { MakeBucketError(std::string err) : runtime_error(err) {} }; @@ -1537,6 +1576,58 @@ class Tests { RemoveObject(bucket_name_, object_name); } } // TestAsyncOperations + + // Regression test for SelectHandler Stats metric parsing: metrics larger + // than INT32_MAX must round-trip as exact long long values (any fallback + // to std::stol on 32-bit Windows LLP64 would truncate them). Uses a + // synthetic Stats event frame, independent of large objects. + void SelectStatsMetrics() { + std::cout << "SelectStatsMetrics()" << std::endl; + + const long long scanned = 5000000000LL; + const long long processed = 6000000000LL; + const long long returned = 7000000000LL; + + std::map headers = { + {":message-type", "event"}, + {":event-type", "Stats"}, + }; + std::string payload = "" + std::to_string(scanned) + + "" + + std::to_string(processed) + + "" + + std::to_string(returned) + ""; + + bool stats_delivered = false; + minio::s3::SelectHandler handler( + [&](minio::s3::SelectResult result) -> bool { + if (result.err) { + throw std::runtime_error("SelectStatsMetrics(): " + + result.err.String()); + } + if (!result.bytes_scanned.has_value() || + *result.bytes_scanned != scanned || + !result.bytes_processed.has_value() || + *result.bytes_processed != processed || + !result.bytes_returned.has_value() || + *result.bytes_returned != returned) { + throw std::runtime_error( + "SelectStatsMetrics(): unexpected metrics"); + } + stats_delivered = true; + return true; + }); + + minio::http::DataFunctionArgs args; + args.datachunk = MakeSelectFrame(headers, payload); + if (!handler.DataFunction(args)) { + throw std::runtime_error("SelectStatsMetrics(): DataFunction failed"); + } + if (!stats_delivered) { + throw std::runtime_error( + "SelectStatsMetrics(): Stats result was not delivered"); + } + } }; // class Tests int main(int /*argc*/, char* /*argv*/[]) { @@ -1596,6 +1687,7 @@ int main(int /*argc*/, char* /*argv*/[]) { tests.SelectObjectContent(); tests.ListenBucketNotification(); tests.TestAsyncOperations(); + tests.SelectStatsMetrics(); return EXIT_SUCCESS; } From 1d9fe92c66cd3238fa6c2237461524956122c91e Mon Sep 17 00:00:00 2001 From: jiuker Date: Thu, 13 Aug 2026 15:44:39 +0800 Subject: [PATCH 4/6] Update tests.cc --- tests/tests.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/tests.cc b/tests/tests.cc index 720b56e0..ae854268 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -133,11 +133,11 @@ std::string MakeSelectFrame(const std::map& headers, std::string prelude = PutUint32BigEndian(total_length) + PutUint32BigEndian(static_cast(headerdata.length())); - std::string prelude_crc = - PutUint32BigEndian(static_cast(utils::CRC32(prelude))); + std::string prelude_crc = PutUint32BigEndian( + static_cast(minio::utils::CRC32(prelude))); std::string message = prelude + prelude_crc + data; - std::string message_crc = - PutUint32BigEndian(static_cast(utils::CRC32(message))); + std::string message_crc = PutUint32BigEndian( + static_cast(minio::utils::CRC32(message))); return message + message_crc; } From b8ba8c8c0ae9c42a27f39d9ad3449bcf0abb3ac1 Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 17 Aug 2026 09:12:18 +0800 Subject: [PATCH 5/6] fix: C++17-compatible RDMA ctx init and binary download test coverage - Replace C++20 designated initializers for minio::rdma::ClientCtx with positional aggregate initialization so MINIO_CPP_RDMA builds under the default C++17 standard (GCC/Clang accepted the designators only as an extension). - Harden the DownloadObject test with newline, 0x1A (Windows Ctrl-Z), and NUL bytes, read back via std::ios::binary, and compare exact bytes. --- include/miniocpp/rdma.h | 12 +++++++----- src/baseclient.cc | 21 ++++++--------------- src/client.cc | 20 ++++++-------------- tests/tests.cc | 14 ++++++++++---- 4 files changed, 29 insertions(+), 38 deletions(-) diff --git a/include/miniocpp/rdma.h b/include/miniocpp/rdma.h index 47266de8..083e26b7 100644 --- a/include/miniocpp/rdma.h +++ b/include/miniocpp/rdma.h @@ -50,11 +50,13 @@ namespace minio::rdma { // Per-request state the RDMA control plane needs to build and sign the S3 // request that carries the token. struct ClientCtx { - // All members carry explicit in-class defaults so designated-initializer - // construction (e.g. `ClientCtx{.bucket=...}`) does not trip - // -Wmissing-field-initializers for the std::string fields we leave - // unspecified at single-shot Put/Get call sites (uploadId/partNumber for - // non-multipart paths, etag/checksum for fields populated by the callee). + // All members carry explicit in-class defaults so aggregate initialization + // can omit trailing members (and spell skipped ones as {}/std::nullopt) + // without tripping -Wmissing-field-initializers for the std::string fields + // left unspecified at single-shot Put/Get call sites (uploadId/partNumber + // for non-multipart paths, etag/checksum populated by the callee). Positional + // init is used rather than C++20 designated initializers so the library + // keeps building under the default C++17 standard. minio::creds::Provider* const provider = nullptr; std::string bucket = {}; std::string object = {}; diff --git a/src/baseclient.cc b/src/baseclient.cc index e7b1a5a4..ad2247cf 100644 --- a/src/baseclient.cc +++ b/src/baseclient.cc @@ -1446,13 +1446,9 @@ Result BaseClient::PutObject(PutObjectApiArgs args) { #ifdef MINIO_CPP_RDMA if (args.rdmaclient != nullptr && args.rdmaclient->Ready()) { - minio::rdma::ClientCtx putCtx = { - .provider = provider_, - .bucket = args.bucket, - .object = args.object, - .url = base_url_, - .region = region, - }; + minio::rdma::ClientCtx putCtx = {provider_, args.bucket, args.object, + {}, std::nullopt, {}, + base_url_, region}; ssize_t ret = rdmaPutWithRetry(args.rdmaclient, &putCtx, args.buf, args.size); @@ -2100,14 +2096,9 @@ Result BaseClient::UploadPart(UploadPartArgs args) { } minio::rdma::ClientCtx putCtx = { - .provider = provider_, - .bucket = args.bucket, - .object = args.object, - .uploadId = args.upload_id, - .partNumber = args.part_number, - .url = base_url_, - .region = region, - .checksum = args.checksum_crc64nvme, + provider_, args.bucket, args.object, + args.upload_id, args.part_number, {}, + base_url_, region, args.checksum_crc64nvme, }; ssize_t ret = diff --git a/src/client.cc b/src/client.cc index 75770ef9..b8474f21 100644 --- a/src/client.cc +++ b/src/client.cc @@ -705,13 +705,9 @@ Result Client::GetObject(GetObjectArgs args) { size <= kRDMAMaxMemoryRegSize && rdma_client.Register(args.buf, size); if (use_rdma) { - minio::rdma::ClientCtx getCtx = { - .provider = provider_, - .bucket = args.bucket, - .object = args.object, - .url = base_url_, - .region = region, - }; + minio::rdma::ClientCtx getCtx = {provider_, args.bucket, args.object, + {}, std::nullopt, {}, + base_url_, region}; // RAII, matching the multipart paths below. rdmaGetWithRetry signs and // sends an HTTP request, and curlpp throws, so a manual Deregister after @@ -1237,13 +1233,9 @@ Result Client::PutObject(PutObjectArgs args) { size <= kRDMAMaxMemoryRegSize && rdma_client.Register(args.buf, size); if (use_rdma) { - minio::rdma::ClientCtx putCtx = { - .provider = provider_, - .bucket = args.bucket, - .object = args.object, - .url = base_url_, - .region = region, - }; + minio::rdma::ClientCtx putCtx = {provider_, args.bucket, args.object, + {}, std::nullopt, {}, + base_url_, region}; // RAII, matching the multipart paths below -- see the GET path for why // a manual Deregister after the call is not enough. diff --git a/tests/tests.cc b/tests/tests.cc index ae55c0a0..f8fb8554 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -371,7 +371,12 @@ class Tests { std::string object_name = RandObjectName(); - std::string data = "DownloadObject()"; + // Binary-safe round-trip: a newline, the Windows text-mode EOF byte + // (0x1A / Ctrl-Z), and a NUL byte must survive download byte-for-byte. + std::string data = "DownloadObject()\n"; + data += static_cast(0x1A); + data += '\0'; + data += "binary-tail"; std::stringstream ss(data); minio::s3::PutObjectArgs args(ss, static_cast(data.length()), 0); args.bucket = bucket_name_; @@ -392,7 +397,7 @@ class Tests { throw std::runtime_error("DownloadObject(): " + resp.error().String()); } - std::ifstream file(filename); + std::ifstream file(filename, std::ios::binary); file.seekg(0, std::ios::end); size_t length = file.tellg(); file.seekg(0, std::ios::beg); @@ -401,8 +406,9 @@ class Tests { file.close(); if (data != std::string(buf, length)) { - throw std::runtime_error("DownloadObject(): expected: " + data + - "; got: " + buf); + throw std::runtime_error( + "DownloadObject(): expected " + std::to_string(data.length()) + + " bytes; got " + std::to_string(length) + " bytes"); } std::filesystem::remove(filename); RemoveObject(bucket_name_, object_name); From 461d4df892f6ae4483a3b9cddec7fbe5d0c1503c Mon Sep 17 00:00:00 2001 From: jiuker Date: Mon, 17 Aug 2026 09:36:40 +0800 Subject: [PATCH 6/6] test: encode a wire-compatible EventStream header block EventStream header sections (per AWS S3 and MinIO) contain only the encoded headers; the prelude's headers-length is the boundary, there is no terminator byte. The appended 0 made the synthetic Select frame non-wire-compatible and masked the parser's reliance on an empty remaining header buffer. --- tests/tests.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/tests.cc b/tests/tests.cc index f8fb8554..e0f2e68f 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -114,7 +114,9 @@ std::string PutUint32BigEndian(unsigned int v) { } // Build a single S3 Select protocol frame (prelude + prelude CRC + headers -// + payload + message CRC) for the given event headers and XML payload. +// + payload + message CRC) for the given event headers and XML payload. The +// headers section is exactly the encoded headers: the wire format (AWS S3 and +// MinIO) has no terminator byte, the prelude's headers-length is the boundary. std::string MakeSelectFrame(const std::map& headers, const std::string& payload) { std::string headerdata; @@ -126,7 +128,6 @@ std::string MakeSelectFrame(const std::map& headers, headerdata += static_cast(value.length() & 0xFF); headerdata += value; } - headerdata += static_cast(0); // header terminator std::string data = headerdata + payload; unsigned int total_length = 16 + static_cast(data.length());