From ed8d8b075c2f107b350d79e092abc885ffc4eff6 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Sat, 1 Aug 2026 12:12:11 +0800 Subject: [PATCH 1/2] chore: prepare 0.3 repository updates --- .asf.yaml | 14 +++++---- .github/workflows/build_and_test.yaml | 8 ++--- CMakeLists.txt | 3 +- CONTRIBUTING.md | 2 +- LICENSE | 10 ------ NOTICE | 5 --- README.md | 6 +--- docs/source/build_system.rst | 2 +- docs/source/building.rst | 31 +++---------------- docs/source/conf.py | 4 +-- docs/source/examples/benchmark.rst | 8 +++-- docs/source/user_guide/global_index.rst | 20 +++++++----- docs/source/user_guide/manifest_cache.rst | 13 +++----- scripts/releasing/README.md | 26 ++++++++-------- scripts/releasing/create_source_release.sh | 2 +- scripts/releasing/release_rc.sh | 2 +- .../format/parquet/file_reader_wrapper.h | 3 +- 17 files changed, 63 insertions(+), 96 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 96daab1b..0a9c4a5a 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -32,7 +32,7 @@ github: enabled_merge_buttons: merge: false squash: true - rebase: true + rebase: false rulesets: - name: "Branch Protection" type: branch @@ -52,17 +52,19 @@ github: required_status_checks: - name: "pre-commit" app_slug: -1 - - name: "gcc-test" - app_slug: -1 - name: "rat-license-check" app_slug: -1 - - name: "test-with-sanitizer" + - name: "asan-ubsan" + app_slug: -1 + - name: "tsan" + app_slug: -1 + - name: "clang-debug" app_slug: -1 - name: "clang-release" app_slug: -1 - - name: "gcc-release" + - name: "gcc-debug" app_slug: -1 - - name: "clang-test" + - name: "gcc-release" app_slug: -1 - name: "gcc8-test" app_slug: -1 diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index a015342b..ab3f22bb 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -51,10 +51,10 @@ jobs: cc: gcc-14 cxx: g++-14 - name: clang-debug - # fetch_depth: '0' # fetch the PR target branch history for clang-tidy - # build_args: >- - # --check_clang_tidy - # --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}" + fetch_depth: '0' # fetch the PR target branch history for clang-tidy + build_args: >- + --check_clang_tidy + --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}" - name: asan-ubsan build_args: --enable_asan --enable_ubsan - name: tsan diff --git a/CMakeLists.txt b/CMakeLists.txt index 52053d4e..1122884b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,6 +64,7 @@ option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF) option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF) option(PAIMON_ENABLE_NETWORK_TESTS "Whether to enable tests that access real remote services over the network" OFF) +option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) option(PAIMON_ENABLE_TANTIVY "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) @@ -485,7 +486,7 @@ include(CMakePackageConfigHelpers) write_basic_package_version_file( "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfigVersion.cmake" VERSION ${PROJECT_VERSION} - COMPATIBILITY AnyNewerVersion) + COMPATIBILITY SameMinorVersion) configure_package_config_file("${CMAKE_CURRENT_SOURCE_DIR}/PaimonConfig.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cabeca59..3e246735 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ Before submitting, please verify: ### Prerequisites - **C++17** compatible compiler (GCC recommended) -- **CMake** ≥ 3.16 +- **CMake** ≥ 3.22 - **Python 3** (for linting scripts and pre-commit) --- diff --git a/LICENSE b/LICENSE index 6e39411a..bc9931be 100644 --- a/LICENSE +++ b/LICENSE @@ -429,16 +429,6 @@ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- -This product includes code from LucenePlusPlus. - -* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory - -Copyright: 2009-2014 Alan Wright. -Home page: https://github.com/luceneplusplus/LucenePlusPlus -License: https://www.apache.org/licenses/LICENSE-2.0 - --------------------------------------------------------------------------------- - This product includes code derived from PyTorch TH simd.h. * SIMD detection code in third_party/roaring_bitmap/roaring.cpp diff --git a/NOTICE b/NOTICE index a12e432f..55104862 100644 --- a/NOTICE +++ b/NOTICE @@ -50,10 +50,5 @@ PalDB -------------------------------------------------------------------------------- -This product includes software from LucenePlusPlus project (Apache 2.0) -Copyright 2009-2014 Alan Wright. - --------------------------------------------------------------------------------- - JindoSDK NextArch C++ Copyright 2024-present Alibaba Cloud. diff --git a/README.md b/README.md index d1140f51..b533ad7b 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,6 @@ It provides native, high-performance, and extensible access to the Paimon lake f Background and documentation are available at [paimon.apache.org](https://paimon.apache.org). -## Status - -Paimon-cpp is currently undergoing repository migration. The original repository is hosted at [github.com/alibaba/paimon-cpp](https://github.com/alibaba/paimon-cpp/), and the codebase is being migrated incrementally to the Apache Paimon community repository. - ## Features Paimon-cpp currently provides: @@ -53,7 +49,7 @@ Paimon-cpp currently provides: > Paimon C++ therefore currently treats the `bitmap` global index type as unsupported. The legacy > implementation remains in the codebase pending migration to the Java-compatible format. -Note: Linux x86_64 and macOS arm64 builds are currently verified. +Note: Only Linux x86_64 builds are currently supported and verified. ## Building diff --git a/docs/source/build_system.rst b/docs/source/build_system.rst index f3b18e54..b2db8103 100644 --- a/docs/source/build_system.rst +++ b/docs/source/build_system.rst @@ -44,7 +44,7 @@ for data format and file system. .. code-block:: cmake - cmake_minimum_required(VERSION 3.16) + cmake_minimum_required(VERSION 3.22) project(MyExample) diff --git a/docs/source/building.rst b/docs/source/building.rst index 8608107a..1041d57b 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -32,9 +32,8 @@ out-of-source. For example, you could create ``paimon-cpp/build`` and invoke Building requires: -* A C++17-enabled compiler. On Linux, gcc 8 and higher should be - sufficient. On macOS, use AppleClang from Xcode Command Line Tools or - LLVM clang from Homebrew. Windows is not supported for now. +* A C++17-enabled compiler. On Linux, gcc 8 and higher should be sufficient. + macOS and Windows are not supported for now. * At least 2GB of RAM for a minimal build, 8GB for a minimal debug build with tests and 16GB for a full build. @@ -46,29 +45,6 @@ On Ubuntu/Debian you can install the requirements with: build-essential \ cmake -On macOS you can install the requirements with: - -.. code-block:: shell - - xcode-select --install - brew install cmake - -The same CMake build options apply on Linux and macOS. If you prefer upstream -LLVM clang instead of AppleClang on macOS, install LLVM and pass the Homebrew -compiler paths when configuring: - -.. code-block:: shell - - brew install llvm - cmake -B build \ - -DCMAKE_C_COMPILER="$(brew --prefix llvm)/bin/clang" \ - -DCMAKE_CXX_COMPILER="$(brew --prefix llvm)/bin/clang++" - -When building with upstream Clang on macOS, Paimon uses Apple SDK libc++ -headers by default to avoid incompatibilities in bundled third-party -dependencies. Pass ``-DPAIMON_USE_APPLE_LIBCXX_WITH_CLANG=OFF`` to disable -this behavior. - We also provide a docker template to help you get started quickly. See in ``.devcontainer`` folder for more details. @@ -149,6 +125,7 @@ boolean flags to ``cmake``. * ``-DPAIMON_ENABLE_JINDO=ON``: Support for Alibaba Jindo filesystems * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. * ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes +* ``-DPAIMON_ENABLE_TANTIVY=ON``: Enable the experimental Tantivy full-text index Rust FFI. Third-party dependency source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -241,7 +218,7 @@ several times with different options if you want to exercise all of them. CMake version requirements ~~~~~~~~~~~~~~~~~~~~~~~~~~ -We support CMake 3.16 and higher. +We support CMake 3.22 and higher. LLVM and Clang Tools ~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/conf.py b/docs/source/conf.py index 92dfa7c1..137c7c70 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -32,8 +32,8 @@ import os import datetime -project = u'C++ Paimon' -copyright = f'2024-{datetime.datetime.now().year} Apache Software Foundation' +project = u'Paimon C++' +copyright = f'2026-{datetime.datetime.now().year} Apache Software Foundation' author = u'Apache Software Foundation' # -- General configuration --------------------------------------------------- diff --git a/docs/source/examples/benchmark.rst b/docs/source/examples/benchmark.rst index ee17be75..d2ff1cde 100644 --- a/docs/source/examples/benchmark.rst +++ b/docs/source/examples/benchmark.rst @@ -54,9 +54,11 @@ specific options below: ``--paimon_option=:;:`` Repeatable table options passed through to Paimon. The default table file - format is ``parquet``; use ``--paimon_option file.format:`` to - override it. For ``BM_PK_Write`` and ``BM_MOR_Read``, ``bucket`` is forced to - ``1``. + format is ``parquet``. Benchmark output supports ``parquet`` and, when + built with ``PAIMON_ENABLE_ORC=ON``, ``orc``; use + ``--paimon_option file.format:parquet`` or + ``--paimon_option file.format:orc`` to select one. For ``BM_PK_Write`` and + ``BM_MOR_Read``, ``bucket`` is forced to ``1``. Examples ======== diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index bc12c3ae..0cb853a0 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -20,12 +20,12 @@ Global Index Global Index is a powerful indexing mechanism for append-only tables. It enables efficient row-level lookups and filtering without full-table scans. -Paimon supports multiple global index types: +Paimon C++ supports the following global index types: -- **Bitmap Index**: A bitmap-based index. Each distinct value is mapped to a compressed bitmap (RoaringBitmap) that records which rows contain that value, enabling extremely fast set membership tests. - **BTree Index**: An efficient index based on multi-level SST files for scalar column lookups. - **Range Bitmap Index**: A range bitmap index optimized for range predicates on ordered scalar columns. Extends the bitmap approach by encoding value ordering, enabling efficient less-than, greater-than, and range conditions. - **Lucene Index**: A full-text search index powered by Lucene++. Supports tokenized text search with multiple modes including match-all, match-any, phrase, prefix, and wildcard queries. +- **Tantivy Full-Text Index (experimental)**: A Rust FFI-based full-text search index. - **Vector Index (Lumina)**: An approximate nearest neighbor (ANN) index powered by Lumina for vector similarity search with configurable distance metrics. Global indexes work on top of Data Evolution tables. To use global indexes, your table must have: @@ -34,12 +34,10 @@ Global indexes work on top of Data Evolution tables. To use global indexes, your - ``'row-tracking.enabled' = 'true'`` - ``'data-evolution.enabled' = 'true'`` -Bitmap Index ------------- +Bitmap Index Compatibility +-------------------------- -A bitmap-based index for Equal and In predicates. Each distinct value in the indexed column -is mapped to a compressed bitmap (RoaringBitmap) that records which rows contain that value. -This allows extremely fast set membership tests. +The current Paimon C++ version does not support bitmap global indexes. BTree Index ----------- @@ -94,6 +92,14 @@ search modes including match-all, match-any, phrase, prefix, and wildcard querie - **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error. +Tantivy Full-Text Index (Experimental) +--------------------------------------- + +The Tantivy full-text index is an experimental Rust FFI-based backend. Enable it at build time +with ``-DPAIMON_ENABLE_TANTIVY=ON``. Its global index identifier is ``tantivy-fulltext``. +Because this backend is experimental, validate its behavior and compatibility for your workload +before production use. + Vector Index (Lumina) --------------------- diff --git a/docs/source/user_guide/manifest_cache.rst b/docs/source/user_guide/manifest_cache.rst index 0e027509..ed01e0f9 100644 --- a/docs/source/user_guide/manifest_cache.rst +++ b/docs/source/user_guide/manifest_cache.rst @@ -22,10 +22,9 @@ Overview -------- paimon-cpp caches raw manifest file bytes at the ``ObjectsFile::Read()`` -layer. The cache uses the public ``Cache`` abstraction and is injected through -``ScanContextBuilder`` or ``ReadContextBuilder``. The cache covers data -manifests, manifest lists, and index manifests because they all read through -``ObjectsFile``. +layer. The cache uses the public ``Cache`` abstraction and is enabled through +``ScanContextBuilder::WithCache()``. The cache covers data manifests, manifest +lists, and index manifests because they all read through ``ObjectsFile``. For repeated ``get``, ``scan``, or batch ``get/scan -f`` requests in the same process, the same snapshot often reads the same manifest files repeatedly. On a @@ -84,10 +83,8 @@ Example: paimon::ScanContextBuilder scan_builder(table_path); scan_builder.WithCache(cache); - paimon::ReadContextBuilder read_builder(table_path); - read_builder.WithCache(cache); - -Passing ``nullptr`` or omitting ``WithCache()`` leaves manifest caching disabled. +Passing ``nullptr`` or omitting ``ScanContextBuilder::WithCache()`` leaves +manifest caching disabled. Future Optimizations -------------------- diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md index d7757a9e..e639bc20 100644 --- a/scripts/releasing/README.md +++ b/scripts/releasing/README.md @@ -47,8 +47,8 @@ Before starting a release: For example, update all version locations and review the diff: ```bash -scripts/releasing/bump_version.py 0.2.2 0.2.3 -scripts/releasing/bump_version.py --check 0.2.3 +scripts/releasing/bump_version.py 0.2.0 0.3.0 +scripts/releasing/bump_version.py --check 0.3.0 ``` ### Signing key setup and security @@ -74,8 +74,8 @@ See the ASF policies for the complete requirements. The release scripts use `vVERSION-rcRC` for release-candidate tags and -`vVERSION` for the final release tag. For example, the first 0.2.3 candidate is -`v0.2.3-rc1`. +`vVERSION` for the final release tag. For example, the first 0.3.0 candidate is +`v0.3.0-rc1`. ## Create a release candidate @@ -88,7 +88,7 @@ workflow to succeed, and imports the artifacts into ASF `dist/dev`: ```bash scripts/releasing/release_rc.sh \ - --version 0.2.3 \ + --version 0.3.0 \ --rc 1 \ --signing-key ASF_GPG_KEY_ID \ --remote upstream @@ -107,9 +107,9 @@ email and must not be used to start a vote. The candidate directory contains: ```text -apache-paimon-cpp-0.2.3-src.tgz -apache-paimon-cpp-0.2.3-src.tgz.asc -apache-paimon-cpp-0.2.3-src.tgz.sha512 +apache-paimon-cpp-0.3.0-src.tgz +apache-paimon-cpp-0.3.0-src.tgz.asc +apache-paimon-cpp-0.3.0-src.tgz.sha512 ``` The wrapper prints a vote-email template. Send it to `dev@paimon.apache.org`. @@ -140,7 +140,7 @@ After its vote is closed, a failed or superseded candidate may be removed from Voters can download and verify an ASF-staged candidate in one command: ```bash -scripts/releasing/verify_release_candidate.sh --version 0.2.3 --rc 1 +scripts/releasing/verify_release_candidate.sh --version 0.3.0 --rc 1 ``` To verify files that were downloaded separately, use an explicitly downloaded @@ -149,14 +149,14 @@ KEYS file so signature verification runs in an isolated GPG home: ```bash scripts/releasing/verify_release_candidate.sh \ --keys-file /path/to/paimon-KEYS \ - apache-paimon-cpp-0.2.3-src.tgz + apache-paimon-cpp-0.3.0-src.tgz ``` The verifier checks: - the SHA-512 checksum and detached OpenPGP signature; - archive path safety, portable filename collisions, file types, permissions, - and the single `paimon-cpp-0.2.3/` root directory; + and the single `paimon-cpp-0.3.0/` root directory; - required `LICENSE`, `NOTICE`, build, and documentation files; - the CMake and documentation versions; - absence of compiled artifacts by filename and file magic; @@ -164,7 +164,7 @@ The verifier checks: - a release build and the test suite from the extracted source archive; and - installation plus compilation and execution of an external CMake consumer. -Pass `--git-ref v0.2.3-rc1` when the Git repository is available to regenerate +Pass `--git-ref v0.3.0-rc1` when the Git repository is available to regenerate the archive from the signed tag and compare it byte-for-byte. `--allow-unsigned`, `--skip-rat`, `--skip-build`, and `--skip-install` exist for @@ -180,7 +180,7 @@ approved candidate: ```bash scripts/releasing/publish_release.sh \ - --version 0.2.3 \ + --version 0.3.0 \ --rc 1 \ --signing-key ASF_GPG_KEY_ID \ --remote upstream \ diff --git a/scripts/releasing/create_source_release.sh b/scripts/releasing/create_source_release.sh index f65ae2b2..125ab579 100755 --- a/scripts/releasing/create_source_release.sh +++ b/scripts/releasing/create_source_release.sh @@ -35,7 +35,7 @@ Usage: create_source_release.sh --version VERSION --git-ref REF [options] Required: - --version VERSION Release version, for example 0.2.3 + --version VERSION Release version, for example 0.3.0 --git-ref REF Commit or signed RC tag to archive Options: diff --git a/scripts/releasing/release_rc.sh b/scripts/releasing/release_rc.sh index 6b5f4058..f943491e 100755 --- a/scripts/releasing/release_rc.sh +++ b/scripts/releasing/release_rc.sh @@ -40,7 +40,7 @@ Usage: release_rc.sh --version VERSION --rc RC --signing-key KEY_ID [options] Required: - --version VERSION Release version, for example 0.2.3 + --version VERSION Release version, for example 0.3.0 --rc RC Release candidate number, for example 1 --signing-key KEY_ID OpenPGP key used for the tag and source artifact diff --git a/src/paimon/format/parquet/file_reader_wrapper.h b/src/paimon/format/parquet/file_reader_wrapper.h index c62c1b96..5dbd3817 100644 --- a/src/paimon/format/parquet/file_reader_wrapper.h +++ b/src/paimon/format/parquet/file_reader_wrapper.h @@ -69,7 +69,8 @@ class FileReaderWrapper { /// @return The next RecordBatch, or nullptr if end of data. Result> Next(); - /// Get the first row number of the previously returned batch. + /// Get the first row number of the previously returned batch. After Next() reaches EOF, + /// returns the next unread row number (the end of the readable range). Result GetPreviousBatchFirstRowNumber() const { return previous_first_row_; } From 75eda16b7f507244a098c5eb90d09e9bb518b8e6 Mon Sep 17 00:00:00 2001 From: "jinli.zjw" Date: Sat, 1 Aug 2026 15:06:02 +0800 Subject: [PATCH 2/2] fix: address v0.3.0 release blockers --- .github/workflows/release_candidate.yaml | 9 ++ docs/source/conf.py | 6 +- docs/source/index.rst | 4 +- include/paimon/utils/special_field_ids.h | 12 +-- scripts/releasing/README.md | 3 +- scripts/releasing/install_smoke/main.cpp | 10 ++- scripts/releasing/tests/test_release_tools.py | 85 ++++++++++++++++++- scripts/releasing/verify_release_candidate.sh | 23 ++++- src/paimon/CMakeLists.txt | 1 - src/paimon/core/utils/special_field_ids.cpp | 31 ------- 10 files changed, 129 insertions(+), 55 deletions(-) delete mode 100644 src/paimon/core/utils/special_field_ids.cpp diff --git a/.github/workflows/release_candidate.yaml b/.github/workflows/release_candidate.yaml index aaba9ec9..28a4b093 100644 --- a/.github/workflows/release_candidate.yaml +++ b/.github/workflows/release_candidate.yaml @@ -122,6 +122,15 @@ jobs: name: source-archive path: release/ci + - name: Install Rust toolchain (tantivy-fts) + shell: bash + run: ci/scripts/setup_rust.sh + + - name: Install HTTP and TLS development dependencies + run: | + sudo apt-get update + sudo apt-get install -y libcurl4-openssl-dev libssl-dev + - name: Verify, build, test, and install shell: bash env: diff --git a/docs/source/conf.py b/docs/source/conf.py index 137c7c70..4b4b7782 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -32,7 +32,7 @@ import os import datetime -project = u'Paimon C++' +project = u'Apache Paimon C++' copyright = f'2026-{datetime.datetime.now().year} Apache Software Foundation' author = u'Apache Software Foundation' @@ -131,7 +131,7 @@ }, ], "logo": { - "text": "Paimon C++", + "text": "Apache Paimon C++", }, "show_version_warning_banner": True, } @@ -143,6 +143,6 @@ "doc_path": "docs/source", } -html_title = f"C++ Paimon" +html_title = "Apache Paimon C++" html_show_sourcelink = False diff --git a/docs/source/index.rst b/docs/source/index.rst index 5c4d83a1..7ad9a1d7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,8 +17,8 @@ .. _implementations: -C++ Paimon Documentation -======================== +Apache Paimon C++ +================= Paimon C++ is a high-performance C++ implementation of Apache Paimon. We aim to provide a native, high-performance and extensible implementation that allows diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h index accf07f5..c4e03f00 100644 --- a/include/paimon/utils/special_field_ids.h +++ b/include/paimon/utils/special_field_ids.h @@ -28,20 +28,20 @@ namespace paimon { class SpecialFieldIds { protected: /// System defined constant for field id boundary. Value: INT32_MAX - 10000 - static const int32_t CPP_FIELD_ID_END = std::numeric_limits::max() - 10000; + inline static constexpr int32_t CPP_FIELD_ID_END = std::numeric_limits::max() - 10000; public: /// Special field ID reserved for sequence number. Value: INT32_MAX - 1 - static const int32_t SEQUENCE_NUMBER = std::numeric_limits::max() - 1; + inline static constexpr int32_t SEQUENCE_NUMBER = std::numeric_limits::max() - 1; /// Special field ID reserved for value kind. Value: INT32_MAX - 2 - static const int32_t VALUE_KIND = std::numeric_limits::max() - 2; + inline static constexpr int32_t VALUE_KIND = std::numeric_limits::max() - 2; /// Special field ID reserved for row kind. Value: INT32_MAX - 3 - static const int32_t ROW_KIND = std::numeric_limits::max() - 3; + inline static constexpr int32_t ROW_KIND = std::numeric_limits::max() - 3; /// Special field ID reserved for row ID. Value: INT32_MAX - 5 - static const int32_t ROW_ID = std::numeric_limits::max() - 5; + inline static constexpr int32_t ROW_ID = std::numeric_limits::max() - 5; /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1 - static const int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1; + inline static constexpr int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1; }; } // namespace paimon diff --git a/scripts/releasing/README.md b/scripts/releasing/README.md index e639bc20..ee558c1e 100644 --- a/scripts/releasing/README.md +++ b/scripts/releasing/README.md @@ -55,8 +55,7 @@ scripts/releasing/bump_version.py --check 0.3.0 Complete signing-key setup well before creating the first release candidate: -- new signing keys must use RSA with at least 2048 bits; ASF recommends 4096 - bits for new keys; +- new signing keys must use RSA with at least 4096 bits; - publish the public key to the global public keyserver network; - append the public key to `dist/release/paimon/KEYS`. Never remove historical keys because they are required to verify archived releases; diff --git a/scripts/releasing/install_smoke/main.cpp b/scripts/releasing/install_smoke/main.cpp index 914adfce..34db4f7e 100644 --- a/scripts/releasing/install_smoke/main.cpp +++ b/scripts/releasing/install_smoke/main.cpp @@ -16,12 +16,18 @@ * limitations under the License. */ +#include +#include #include #include "paimon/status.h" +#include "paimon/utils/special_field_ids.h" int main() { const paimon::Status status = paimon::Status::Invalid("install smoke test"); - return !status.ok() && status.ToString().find("install smoke test") != std::string::npos ? 0 - : 1; + const int32_t* row_id = &paimon::SpecialFieldIds::ROW_ID; + const bool valid_status = + !status.ok() && status.ToString().find("install smoke test") != std::string::npos; + const bool valid_row_id = *row_id == std::numeric_limits::max() - 5; + return valid_status && valid_row_id ? 0 : 1; } diff --git a/scripts/releasing/tests/test_release_tools.py b/scripts/releasing/tests/test_release_tools.py index e3dbdacf..d1b4eeaf 100644 --- a/scripts/releasing/tests/test_release_tools.py +++ b/scripts/releasing/tests/test_release_tools.py @@ -19,13 +19,14 @@ import hashlib import io import json +import os import subprocess import sys import tarfile import tempfile import unittest from pathlib import Path -from typing import List, Optional, Tuple +from typing import Dict, List, Optional, Tuple RELEASING_DIR = Path(__file__).resolve().parents[1] @@ -90,6 +91,17 @@ def create_verifier_archive(self, directory: Path) -> Path: ), ".github/.rat-excludes": b"", "scripts/releasing/create_source_release.sh": b"#!/usr/bin/env bash\n", + "ci/scripts/build_paimon.sh": b"""#!/usr/bin/env bash +set -euo pipefail +source_root=$(cd "$(dirname "$0")/../.." && pwd) +[[ $# == 5 ]] +[[ $1 == --source_dir ]] +[[ $2 == "${source_root}" ]] +[[ $3 == --build_type ]] +[[ $4 == Release ]] +[[ $5 == --install_smoke ]] +[[ ${PAIMON_BUILD_JOBS} == 7 ]] +""", } with tarfile.open(artifact, mode="w:gz") as archive: root = tarfile.TarInfo("paimon-cpp-1.2.3/") @@ -109,20 +121,27 @@ def create_verifier_archive(self, directory: Path) -> Path: return artifact def run_verifier( - self, artifact: Path, *, expected_returncode: int = 0 + self, + artifact: Path, + *, + options: Optional[List[str]] = None, + env: Optional[Dict[str, str]] = None, + expected_returncode: int = 0, ) -> subprocess.CompletedProcess: + if options is None: + options = ["--skip-rat", "--skip-build"] result = subprocess.run( [ "bash", str(RELEASE_VERIFIER), "--allow-unsigned", - "--skip-rat", - "--skip-build", + *options, str(artifact), ], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env, check=False, ) self.assertEqual( @@ -132,6 +151,24 @@ def run_verifier( ) return result + def create_fake_rat(self, directory: Path) -> Tuple[Path, Dict[str, str]]: + rat_jar = directory / "apache-rat.jar" + rat_jar.touch() + bin_dir = directory / "bin" + bin_dir.mkdir() + java = bin_dir / "java" + java.write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "printf '%s Unknown Licenses\\n' \"${FAKE_RAT_UNKNOWN_COUNT:?}\"\n" + "printf '%s\\n' 'Files with unapproved licenses:'\n", + encoding="utf-8", + ) + java.chmod(0o755) + env = os.environ.copy() + env["PATH"] = f"{bin_dir}{os.pathsep}{env['PATH']}" + return rat_jar, env + def test_archive_validator_accepts_regular_archive(self) -> None: with tempfile.TemporaryDirectory() as temp: artifact = Path(temp) / "valid.tgz" @@ -237,6 +274,46 @@ def test_verifier_rejects_multiple_checksum_lines(self) -> None: result.stderr, ) + def test_verifier_invokes_build_script_with_named_arguments(self) -> None: + with tempfile.TemporaryDirectory() as temp: + artifact = self.create_verifier_archive(Path(temp)) + result = self.run_verifier( + artifact, + options=["--skip-rat", "--jobs", "7"], + ) + self.assertIn("Release build and tests: valid", result.stdout) + self.assertIn("Install and consumer smoke test: valid", result.stdout) + + def test_verifier_accepts_zero_unknown_licenses(self) -> None: + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + artifact = self.create_verifier_archive(directory) + rat_jar, env = self.create_fake_rat(directory) + env["FAKE_RAT_UNKNOWN_COUNT"] = "0" + result = self.run_verifier( + artifact, + options=["--rat-jar", str(rat_jar), "--skip-build"], + env=env, + ) + self.assertIn("Apache RAT: valid", result.stdout) + + def test_verifier_rejects_unknown_licenses(self) -> None: + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + artifact = self.create_verifier_archive(directory) + rat_jar, env = self.create_fake_rat(directory) + env["FAKE_RAT_UNKNOWN_COUNT"] = "1" + result = self.run_verifier( + artifact, + options=["--rat-jar", str(rat_jar), "--skip-build"], + env=env, + expected_returncode=1, + ) + self.assertIn( + "Apache RAT found 1 files with unknown licenses", + result.stderr, + ) + def create_version_tree(self, root: Path) -> None: (root / "docs/source/_static").mkdir(parents=True) (root / "CMakeLists.txt").write_text( diff --git a/scripts/releasing/verify_release_candidate.sh b/scripts/releasing/verify_release_candidate.sh index dee7d812..99e7df0a 100755 --- a/scripts/releasing/verify_release_candidate.sh +++ b/scripts/releasing/verify_release_candidate.sh @@ -339,9 +339,18 @@ if [[ "${SKIP_RAT}" == false ]]; then -E "${SOURCE_DIR}/.github/.rat-excludes" \ -d "${SOURCE_DIR}" >"${RAT_REPORT}" - if grep -Eq 'Files with unapproved licenses:[[:space:]]*[1-9]' "${RAT_REPORT}"; then + UNKNOWN_LICENSE_COUNT=$( + awk '/^[[:space:]]*[0-9]+[[:space:]]+Unknown Licenses[[:space:]]*$/ { + print $1 + }' "${RAT_REPORT}" + ) + if [[ ! "${UNKNOWN_LICENSE_COUNT}" =~ ^[0-9]+$ ]]; then cat "${RAT_REPORT}" - fail "Apache RAT found files with unapproved licenses" + fail "could not determine the Apache RAT unknown license count" + fi + if [[ "${UNKNOWN_LICENSE_COUNT}" != "0" ]]; then + cat "${RAT_REPORT}" + fail "Apache RAT found ${UNKNOWN_LICENSE_COUNT} files with unknown licenses" fi echo "Apache RAT: valid" else @@ -353,9 +362,15 @@ if [[ "${SKIP_BUILD}" == false ]]; then if [[ "${SKIP_INSTALL}" == true ]]; then INSTALL_SMOKE=false fi + BUILD_ARGS=( + --source_dir "${SOURCE_DIR}" + --build_type Release + ) + if [[ "${INSTALL_SMOKE}" == true ]]; then + BUILD_ARGS+=(--install_smoke) + fi PAIMON_BUILD_JOBS="${JOBS}" \ - "${SOURCE_DIR}/ci/scripts/build_paimon.sh" \ - "${SOURCE_DIR}" false false Release "${INSTALL_SMOKE}" + "${SOURCE_DIR}/ci/scripts/build_paimon.sh" "${BUILD_ARGS[@]}" echo "Release build and tests: valid" if [[ "${INSTALL_SMOKE}" == true ]]; then echo "Install and consumer smoke test: valid" diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index f80c3ee2..ce2f5ddf 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -403,7 +403,6 @@ set(PAIMON_CORE_SRCS core/utils/partition_path_utils.cpp core/utils/primary_key_table_utils.cpp core/utils/snapshot_manager.cpp - core/utils/special_field_ids.cpp core/utils/tag_manager.cpp) add_paimon_lib(paimon diff --git a/src/paimon/core/utils/special_field_ids.cpp b/src/paimon/core/utils/special_field_ids.cpp deleted file mode 100644 index 7b07837b..00000000 --- a/src/paimon/core/utils/special_field_ids.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include "paimon/utils/special_field_ids.h" - -namespace paimon { - -const int32_t SpecialFieldIds::CPP_FIELD_ID_END; -const int32_t SpecialFieldIds::SEQUENCE_NUMBER; -const int32_t SpecialFieldIds::VALUE_KIND; -const int32_t SpecialFieldIds::ROW_KIND; -const int32_t SpecialFieldIds::ROW_ID; -const int32_t SpecialFieldIds::INDEX_SCORE; - -} // namespace paimon