From 2516d424653b1f436bf6c14db496808f76f0fbc9 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Sun, 17 May 2026 23:10:07 +0300 Subject: [PATCH 01/13] first raw version of a debian package --- CMakeLists.txt | 1 + CMakePresets.json | 16 ++++ cmake/PackSDK.cmake | 19 +++-- cmake/common.cmake | 12 ++- cmake/install.cmake | 85 +++++++++++++------ cmake/ydb-cpp-sdk-config.cmake.in | 4 + library/cpp/logger/CMakeLists.txt | 1 + plugins/metrics/CMakeLists.txt | 2 +- plugins/metrics/otel/CMakeLists.txt | 4 +- plugins/metrics/otel/metrics.cpp | 3 +- plugins/trace/CMakeLists.txt | 2 +- plugins/trace/otel/CMakeLists.txt | 4 +- plugins/trace/otel/trace.cpp | 27 +++--- scripts/test_deb_packages.sh | 25 ++++++ tests/deb_package/Dockerfile | 39 +++++++++ tests/deb_package/test_project/CMakeLists.txt | 16 ++++ tests/deb_package/test_project/main.cpp | 14 +++ 17 files changed, 222 insertions(+), 52 deletions(-) create mode 100755 scripts/test_deb_packages.sh create mode 100644 tests/deb_package/Dockerfile create mode 100644 tests/deb_package/test_project/CMakeLists.txt create mode 100644 tests/deb_package/test_project/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6df450c510c..61fa2137f1f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,7 @@ if (YDB_SDK_INSTALL) CONFIGURATIONS RELEASE NAMESPACE YDB-CPP-SDK:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk/release + COMPONENT libydb-cpp ) configure_package_config_file( ${YDB_SDK_SOURCE_DIR}/cmake/ydb-cpp-sdk-config.cmake.in diff --git a/CMakePresets.json b/CMakePresets.json index d33c11b6811..4186a63ab6b 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -65,6 +65,22 @@ "inherits": ["release-base", "clang-toolchain"], "displayName": "Default Release Config (Clang)" }, + { + "name": "package-deb-clang", + "inherits": ["release-base", "clang-toolchain"], + "displayName": "Package Config (Clang) for .deb", + "binaryDir": "${sourceDir}/build-deb", + "cacheVariables": { + "YDB_SDK_INSTALL": "ON", + "YDB_SDK_ENABLE_OTEL_METRICS": "ON", + "YDB_SDK_ENABLE_OTEL_TRACE": "ON", + "YDB_SDK_EXAMPLES": "OFF", + "BUILD_SHARED_LIBS": "OFF", + "CMAKE_EXE_LINKER_FLAGS": "", + "CMAKE_SHARED_LINKER_FLAGS": "", + "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/base64;~/ydb_deps/brotli;~/ydb_deps/jwt-cpp;~/ydb_deps/opentelemetry-cpp-install" + } + }, { "name": "release-gcc", "inherits": ["release-base", "gcc-toolchain"], diff --git a/cmake/PackSDK.cmake b/cmake/PackSDK.cmake index c500eeebf9e..9856d8e30aa 100644 --- a/cmake/PackSDK.cmake +++ b/cmake/PackSDK.cmake @@ -7,17 +7,26 @@ set(CPACK_PACKAGE_DESCRIPTION with strict consistency and ACID transactions." ) -set(CPACK_PACKAGE_NAME "ydb-cpp-sdk-dev") +set(CPACK_PACKAGE_NAME "ydb-cpp-sdk") set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_GENERATOR "DEB") set(CPACK_PACKAGE_VERSION "${YDB_SDK_VERSION}") set(CPACK_PACKAGE_HOMEPAGE_URL "https://ydb.tech") set(CPACK_PACKAGE_CONTACT "YDB Team ") # TODO set(CPACK_RESOURCE_FILE_LICENSE "${YDB_SDK_SOURCE_DIR}/LICENSE") -# TODO add more parameters -# TODO specify package dependencies -# set(CPACK_DEBIAN_PACKAGE_DEPENDS "liblz4-dev libdouble-conversion-dev libssl-dev") +set(CPACK_DEB_COMPONENT_INSTALL ON) +set(CPACK_COMPONENTS_ALL libydb-cpp libydb-cpp-iam libydb-cpp-otel-metrics libydb-cpp-otel-tracing) + +set(CPACK_DEBIAN_LIBYDB_CPP_PACKAGE_NAME "libydb-cpp-dev") + +set(CPACK_DEBIAN_LIBYDB_CPP_IAM_PACKAGE_NAME "libydb-cpp-iam-dev") +set(CPACK_DEBIAN_LIBYDB_CPP_IAM_PACKAGE_DEPENDS "libydb-cpp-dev (= ${YDB_SDK_VERSION})") + +set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_METRICS_PACKAGE_NAME "libydb-cpp-otel-metrics-dev") +set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_METRICS_PACKAGE_DEPENDS "libydb-cpp-dev (= ${YDB_SDK_VERSION})") + +set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_TRACING_PACKAGE_NAME "libydb-cpp-otel-tracing-dev") +set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_TRACING_PACKAGE_DEPENDS "libydb-cpp-dev (= ${YDB_SDK_VERSION})") -# TODO use lintian in CI to check deb package include(CPack) diff --git a/cmake/common.cmake b/cmake/common.cmake index bd017425190..8a473b0ab06 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -175,7 +175,17 @@ endfunction() function(_ydb_sdk_make_client_component CmpName Tgt) add_library(YDB-CPP-SDK::${CmpName} ALIAS ${Tgt}) - _ydb_sdk_install_targets(TARGETS ${Tgt} ${ARGN}) + if (CmpName STREQUAL "Iam" OR CmpName STREQUAL "IamPrivate") + set(PKG_COMP_NAME "libydb-cpp-iam") + elseif (CmpName STREQUAL "OpenTelemetryMetrics") + set(PKG_COMP_NAME "libydb-cpp-otel-metrics") + elseif (CmpName STREQUAL "OpenTelemetryTrace") + set(PKG_COMP_NAME "libydb-cpp-otel-tracing") + else() + set(PKG_COMP_NAME "libydb-cpp") + endif() + + _ydb_sdk_install_targets(TARGETS ${Tgt} ${ARGN} COMPONENT ${PKG_COMP_NAME}) set(YDB-CPP-SDK_AVAILABLE_COMPONENTS ${YDB-CPP-SDK_AVAILABLE_COMPONENTS} ${CmpName} CACHE INTERNAL "") set(YDB-CPP-SDK_COMPONENT_TARGETS ${YDB-CPP-SDK_COMPONENT_TARGETS} ${Tgt} CACHE INTERNAL "") endfunction() diff --git a/cmake/install.cmake b/cmake/install.cmake index 2c1334f01bd..f56b57ea678 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -2,14 +2,18 @@ function(_ydb_sdk_install_targets) if (NOT YDB_SDK_INSTALL) return() endif() + set(oneValueArgs COMPONENT) set(multiValueArgs TARGETS) cmake_parse_arguments( - ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}" + ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) if (NOT ARG_TARGETS) message(FATAL_ERROR "No TARGETS given for install") return() endif() + if (NOT ARG_COMPONENT) + set(ARG_COMPONENT "libydb-cpp") + endif() foreach (ITEM_TARGET IN LISTS ARG_TARGETS) if(NOT TARGET ${ITEM_TARGET}) message(FATAL_ERROR "${ITEM_TARGET} is not target. You should use only targets") @@ -19,9 +23,9 @@ function(_ydb_sdk_install_targets) install(TARGETS ${ARG_TARGETS} EXPORT ydb-cpp-sdk-targets CONFIGURATIONS RELEASE - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${ARG_COMPONENT} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT ${ARG_COMPONENT} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT ${ARG_COMPONENT} INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} ${CMAKE_INSTALL_INCLUDEDIR}/__ydb_sdk_special_headers @@ -32,10 +36,10 @@ function(_ydb_sdk_directory_install) if (NOT ${YDB_SDK_INSTALL}) return() endif() - set(oneValueArgs DESTINATION PATTERN) + set(oneValueArgs DESTINATION PATTERN COMPONENT EXCLUDE) set(multiValueArgs FILES DIRECTORY) cmake_parse_arguments( - ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" "${ARGN}" + ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) if (NOT ARG_DESTINATION) message(FATAL_ERROR "No DESTINATION for install") @@ -46,13 +50,18 @@ function(_ydb_sdk_directory_install) if (ARG_FILES AND ARG_DIRECTORY) message(FATAL_ERROR "FILES and DIRECTORY are mutually exclusive install arguments") endif() + if (NOT ARG_COMPONENT) + set(ARG_COMPONENT "libydb-cpp") + endif() if (ARG_FILES) - install(FILES ${ARG_FILES} DESTINATION ${ARG_DESTINATION}) + install(FILES ${ARG_FILES} DESTINATION ${ARG_DESTINATION} COMPONENT ${ARG_COMPONENT}) else() if (DEFINED ARG_PATTERN) - install(DIRECTORY ${ARG_DIRECTORY} DESTINATION ${ARG_DESTINATION} USE_SOURCE_PERMISSIONS FILES_MATCHING PATTERN ${ARG_PATTERN}) + install(DIRECTORY ${ARG_DIRECTORY} DESTINATION ${ARG_DESTINATION} USE_SOURCE_PERMISSIONS COMPONENT ${ARG_COMPONENT} FILES_MATCHING PATTERN ${ARG_PATTERN}) + elseif (DEFINED ARG_EXCLUDE) + install(DIRECTORY ${ARG_DIRECTORY} DESTINATION ${ARG_DESTINATION} USE_SOURCE_PERMISSIONS COMPONENT ${ARG_COMPONENT} PATTERN ${ARG_EXCLUDE} EXCLUDE) else() - install(DIRECTORY ${ARG_DIRECTORY} DESTINATION ${ARG_DESTINATION} USE_SOURCE_PERMISSIONS) + install(DIRECTORY ${ARG_DIRECTORY} DESTINATION ${ARG_DESTINATION} USE_SOURCE_PERMISSIONS COMPONENT ${ARG_COMPONENT}) endif() endif() endfunction() @@ -61,27 +70,47 @@ function(_ydb_sdk_install_headers ArgIncludeDir) if (NOT ${YDB_SDK_INSTALL}) return() endif() - _ydb_sdk_directory_install(DIRECTORY ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk - DESTINATION ${ArgIncludeDir} + set(oneValueArgs COMPONENT) + set(multiValueArgs DIRECTORY) + cmake_parse_arguments( + ARG "${option}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} ) - - file(STRINGS ${YDB_SDK_SOURCE_DIR}/cmake/public_headers.txt PublicHeaders) - file(STRINGS ${YDB_SDK_SOURCE_DIR}/cmake/protos_public_headers.txt ProtosPublicHeaders) - if (NOT MSVC) - list(REMOVE_ITEM PublicHeaders library/cpp/deprecated/atomic/atomic_win.h) + if (NOT ARG_COMPONENT) + set(ARG_COMPONENT "libydb-cpp") endif() - foreach(HeaderPath ${PublicHeaders}) - get_filename_component(RelInstallPath ${HeaderPath} DIRECTORY) - _ydb_sdk_directory_install(FILES - ${YDB_SDK_SOURCE_DIR}/${HeaderPath} - DESTINATION ${ArgIncludeDir}/__ydb_sdk_special_headers/${RelInstallPath} + + if (ARG_DIRECTORY) + _ydb_sdk_directory_install(DIRECTORY ${ARG_DIRECTORY} + DESTINATION ${ArgIncludeDir} + COMPONENT ${ARG_COMPONENT} ) - endforeach() - foreach(HeaderPath ${ProtosPublicHeaders}) - get_filename_component(RelInstallPath ${HeaderPath} DIRECTORY) - _ydb_sdk_directory_install(FILES - ${YDB_SDK_BINARY_DIR}/${HeaderPath} - DESTINATION ${ArgIncludeDir}/__ydb_sdk_special_headers/${RelInstallPath} + else() + _ydb_sdk_directory_install(DIRECTORY ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk + DESTINATION ${ArgIncludeDir} + COMPONENT ${ARG_COMPONENT} + EXCLUDE "open_telemetry" ) - endforeach() + + file(STRINGS ${YDB_SDK_SOURCE_DIR}/cmake/public_headers.txt PublicHeaders) + file(STRINGS ${YDB_SDK_SOURCE_DIR}/cmake/protos_public_headers.txt ProtosPublicHeaders) + if (NOT MSVC) + list(REMOVE_ITEM PublicHeaders library/cpp/deprecated/atomic/atomic_win.h) + endif() + foreach(HeaderPath ${PublicHeaders}) + get_filename_component(RelInstallPath ${HeaderPath} DIRECTORY) + _ydb_sdk_directory_install(FILES + ${YDB_SDK_SOURCE_DIR}/${HeaderPath} + DESTINATION ${ArgIncludeDir}/__ydb_sdk_special_headers/${RelInstallPath} + COMPONENT ${ARG_COMPONENT} + ) + endforeach() + foreach(HeaderPath ${ProtosPublicHeaders}) + get_filename_component(RelInstallPath ${HeaderPath} DIRECTORY) + _ydb_sdk_directory_install(FILES + ${YDB_SDK_BINARY_DIR}/${HeaderPath} + DESTINATION ${ArgIncludeDir}/__ydb_sdk_special_headers/${RelInstallPath} + COMPONENT ${ARG_COMPONENT} + ) + endforeach() + endif() endfunction() diff --git a/cmake/ydb-cpp-sdk-config.cmake.in b/cmake/ydb-cpp-sdk-config.cmake.in index ba1c144f0a1..3a1c8299332 100644 --- a/cmake/ydb-cpp-sdk-config.cmake.in +++ b/cmake/ydb-cpp-sdk-config.cmake.in @@ -36,6 +36,10 @@ if (@YDB_SDK_USE_RAPID_JSON@) ) endif() +if (@YDB_SDK_ENABLE_OTEL_METRICS@ OR @YDB_SDK_ENABLE_OTEL_TRACE@) + find_package(opentelemetry-cpp REQUIRED) +endif() + include("${CMAKE_CURRENT_LIST_DIR}/release/ydb-cpp-sdk-targets.cmake") function(_find_ydb_sdk_component CompName) diff --git a/library/cpp/logger/CMakeLists.txt b/library/cpp/logger/CMakeLists.txt index 1047043d0fa..bb0d74e702b 100644 --- a/library/cpp/logger/CMakeLists.txt +++ b/library/cpp/logger/CMakeLists.txt @@ -35,6 +35,7 @@ target_link_libraries(logger.global PUBLIC yutil json + logger PRIVATE enum_serialization_runtime ) diff --git a/plugins/metrics/CMakeLists.txt b/plugins/metrics/CMakeLists.txt index 2bcde514095..9b5f5d8505b 100644 --- a/plugins/metrics/CMakeLists.txt +++ b/plugins/metrics/CMakeLists.txt @@ -1,3 +1,3 @@ if (YDB_SDK_ENABLE_OTEL_METRICS) - add_subdirectory(otel EXCLUDE_FROM_ALL) + add_subdirectory(otel) endif() \ No newline at end of file diff --git a/plugins/metrics/otel/CMakeLists.txt b/plugins/metrics/otel/CMakeLists.txt index fe6bcf4ea04..6875ce3a672 100644 --- a/plugins/metrics/otel/CMakeLists.txt +++ b/plugins/metrics/otel/CMakeLists.txt @@ -1,7 +1,7 @@ _ydb_sdk_add_library(open_telemetry_metrics) target_sources(open_telemetry_metrics PRIVATE - src/metrics.cpp + metrics.cpp ) target_include_directories(open_telemetry_metrics PUBLIC $ @@ -15,4 +15,4 @@ target_link_libraries(open_telemetry_metrics PUBLIC ) _ydb_sdk_make_client_component(OpenTelemetryMetrics open_telemetry_metrics) -_ydb_sdk_install_headers(${CMAKE_INSTALL_INCLUDEDIR} DIRECTORY include/) \ No newline at end of file +_ydb_sdk_directory_install(FILES ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/open_telemetry/metrics.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/ydb-cpp-sdk/open_telemetry COMPONENT libydb-cpp-otel-metrics) \ No newline at end of file diff --git a/plugins/metrics/otel/metrics.cpp b/plugins/metrics/otel/metrics.cpp index 22bfa284107..38705474965 100644 --- a/plugins/metrics/otel/metrics.cpp +++ b/plugins/metrics/otel/metrics.cpp @@ -145,8 +145,9 @@ class TOtelMetricRegistry : public IMetricRegistry { histogramConfig->boundaries_ = buckets; auto view = std::make_unique( + name, std::string{}, - std::string{}, + unit, sdk::metrics::AggregationType::kHistogram, histogramConfig ); diff --git a/plugins/trace/CMakeLists.txt b/plugins/trace/CMakeLists.txt index c1977028a0d..bfa3fe8e27e 100644 --- a/plugins/trace/CMakeLists.txt +++ b/plugins/trace/CMakeLists.txt @@ -1,3 +1,3 @@ if (YDB_SDK_ENABLE_OTEL_TRACE) - add_subdirectory(otel EXCLUDE_FROM_ALL) + add_subdirectory(otel) endif() \ No newline at end of file diff --git a/plugins/trace/otel/CMakeLists.txt b/plugins/trace/otel/CMakeLists.txt index c1cec305ec3..37783bb5cfe 100644 --- a/plugins/trace/otel/CMakeLists.txt +++ b/plugins/trace/otel/CMakeLists.txt @@ -1,6 +1,6 @@ _ydb_sdk_add_library(open_telemetry_trace) target_sources(open_telemetry_trace PRIVATE - src/trace.cpp + trace.cpp ) target_include_directories(open_telemetry_trace PUBLIC $ @@ -13,4 +13,4 @@ target_link_libraries(open_telemetry_trace PUBLIC ) _ydb_sdk_make_client_component(OpenTelemetryTrace open_telemetry_trace) -_ydb_sdk_install_headers(${CMAKE_INSTALL_INCLUDEDIR} DIRECTORY include/) \ No newline at end of file +_ydb_sdk_directory_install(FILES ${YDB_SDK_SOURCE_DIR}/include/ydb-cpp-sdk/open_telemetry/trace.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/ydb-cpp-sdk/open_telemetry COMPONENT libydb-cpp-otel-tracing) \ No newline at end of file diff --git a/plugins/trace/otel/trace.cpp b/plugins/trace/otel/trace.cpp index f3095dd14d4..a606b451847 100644 --- a/plugins/trace/otel/trace.cpp +++ b/plugins/trace/otel/trace.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -60,24 +61,24 @@ class TOtelSpan : public ISpan { Span_->End(); } - void SetAttribute(const std::string& key, const std::string& value) override { - Span_->SetAttribute(key, value); + void SetAttribute(std::string_view key, std::string_view value) override { + Span_->SetAttribute(otel_nostd::string_view(key.data(), key.size()), otel_nostd::string_view(value.data(), value.size())); } - void SetAttribute(const std::string& key, int64_t value) override { - Span_->SetAttribute(key, value); + void SetAttribute(std::string_view key, int64_t value) override { + Span_->SetAttribute(otel_nostd::string_view(key.data(), key.size()), value); } - void AddEvent(const std::string& name, const std::map& attributes) override { - if (attributes.empty()) { - Span_->AddEvent(name); + void AddEvent(std::string_view name, TAttributes attributes = {}) override { + if (attributes.size() == 0) { + Span_->AddEvent(otel_nostd::string_view(name.data(), name.size())); } else { std::vector> attrs; attrs.reserve(attributes.size()); for (const auto& [k, v] : attributes) { - attrs.emplace_back(otel_nostd::string_view(k), otel_common::AttributeValue(otel_nostd::string_view(v))); + attrs.emplace_back(otel_nostd::string_view(k.data(), k.size()), otel_common::AttributeValue(otel_nostd::string_view(v.data(), v.size()))); } - Span_->AddEvent(name, attrs); + Span_->AddEvent(otel_nostd::string_view(name.data(), name.size()), otel_common::MakeAttributes(attrs)); } } @@ -85,8 +86,8 @@ class TOtelSpan : public ISpan { return std::make_unique(Span_); } - void SetStatus(ESpanStatus status, const std::string& description) override { - Span_->SetStatus(MapSpanStatus(status), description); + void SetStatus(ESpanStatus status, std::string_view description = {}) override { + Span_->SetStatus(MapSpanStatus(status), otel_nostd::string_view(description.data(), description.size())); } private: @@ -113,6 +114,10 @@ class TOtelTracer : public ITracer { return std::make_shared(Tracer_->StartSpan(name, options)); } + std::string GetCurrentTraceparent() const override { + return ""; // Stub implementation, OpenTelemetry doesn't natively expose this through the Tracer easily + } + private: otel_nostd::shared_ptr Tracer_; }; diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh new file mode 100755 index 00000000000..5f6b3de13cc --- /dev/null +++ b/scripts/test_deb_packages.sh @@ -0,0 +1,25 @@ +#!/bin/bash +set -e + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +DEB_DIR=$(realpath "$1") +SCRIPT_DIR=$(dirname "$(realpath "$0")") +TEST_DIR=$(realpath "$SCRIPT_DIR/../tests/deb_package") + +echo "Building test Docker image..." +docker build --network host -t ydb-cpp-sdk-deb-test "$TEST_DIR" + +YDB_DEPS_DIR=$(realpath "$SCRIPT_DIR/../../ydb_deps") + +echo "Running test container..." +docker run --rm --network host \ + -v "$DEB_DIR:/deb_packages:ro" \ + -v "$YDB_DEPS_DIR:/ydb_deps:ro" \ + ydb-cpp-sdk-deb-test \ + bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/base64;/ydb_deps/brotli;/ydb_deps/jwt-cpp' && make && ./test_app" + +echo "Test successful!" \ No newline at end of file diff --git a/tests/deb_package/Dockerfile b/tests/deb_package/Dockerfile new file mode 100644 index 00000000000..3c78acbfc24 --- /dev/null +++ b/tests/deb_package/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + git \ + libidn11-dev \ + libssl-dev \ + zlib1g-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libgrpc++-dev \ + protobuf-compiler-grpc \ + libbrotli-dev \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + libxxhash-dev \ + libsnappy-dev \ + libdouble-conversion-dev \ + libgtest-dev \ + libre2-dev \ + libc-ares-dev \ + rapidjson-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN git clone -b v1.12.0 https://github.com/open-telemetry/opentelemetry-cpp.git /tmp/otel && \ + cd /tmp/otel && \ + mkdir build && cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DWITH_OTLP_HTTP=OFF -DWITH_OTLP_GRPC=OFF -DWITH_PROMETHEUS=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \ + make -j$(nproc) && \ + make install && \ + rm -rf /tmp/otel + +COPY test_project /test_project +WORKDIR /test_project \ No newline at end of file diff --git a/tests/deb_package/test_project/CMakeLists.txt b/tests/deb_package/test_project/CMakeLists.txt new file mode 100644 index 00000000000..af4830c4fe0 --- /dev/null +++ b/tests/deb_package/test_project/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.10) +project(test_ydb_cpp_sdk) + +set(CMAKE_CXX_STANDARD 20) + +find_package(ydb-cpp-sdk REQUIRED) + +add_executable(test_app main.cpp) + +target_link_libraries(test_app + PRIVATE + YDB-CPP-SDK::Driver + YDB-CPP-SDK::Iam + YDB-CPP-SDK::OpenTelemetryMetrics + YDB-CPP-SDK::OpenTelemetryTrace +) \ No newline at end of file diff --git a/tests/deb_package/test_project/main.cpp b/tests/deb_package/test_project/main.cpp new file mode 100644 index 00000000000..afb83a28748 --- /dev/null +++ b/tests/deb_package/test_project/main.cpp @@ -0,0 +1,14 @@ +#include +#include + +#include + +int main() { + auto config = NYdb::TDriverConfig() + .SetEndpoint("localhost:2136") + .SetDatabase("/local") + .SetCredentialsProviderFactory(NYdb::CreateIamCredentialsProviderFactory({})); + + std::cout << "Successfully instantiated driver configuration with IAM credentials provider." << std::endl; + return 0; +} \ No newline at end of file From 95ad9ede02dbd33d4b06c865075b88f318a0cd5a Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Mon, 18 May 2026 13:45:31 +0300 Subject: [PATCH 02/13] proper functioning .deb packages for the sdk and plugins --- CMakeLists.txt | 70 ++++++++++++++ cmake/PackSDK.cmake | 10 ++ cmake/common.cmake | 106 ++++++++++++++++++++- cmake/external_libs.cmake | 27 +++++- cmake/global_flags.cmake | 2 +- cmake/install.cmake | 6 ++ debian/libydb-cpp-dev.install | 4 + debian/libydb-cpp-iam-dev.install | 1 + debian/libydb-cpp-otel-metrics-dev.install | 2 + debian/libydb-cpp-otel-tracing-dev.install | 2 + scripts/generate-debian-directory.sh | 35 +++++++ scripts/test_deb_packages.sh | 2 +- scripts/test_dpkg_buildpackage.sh | 66 +++++++++++++ 13 files changed, 327 insertions(+), 6 deletions(-) create mode 100644 debian/libydb-cpp-dev.install create mode 100644 debian/libydb-cpp-iam-dev.install create mode 100644 debian/libydb-cpp-otel-metrics-dev.install create mode 100644 debian/libydb-cpp-otel-tracing-dev.install create mode 100755 scripts/generate-debian-directory.sh create mode 100755 scripts/test_dpkg_buildpackage.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 61fa2137f1f..f4177996491 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -73,6 +73,76 @@ if (YDB_SDK_TESTS) endif() if (YDB_SDK_INSTALL) + function(_ydb_sdk_create_package_library PackageProp TargetName ComponentName) + foreach(Tgt IN LISTS YDB_CPP_${PackageProp}_COMPONENT_TARGETS) + _ydb_sdk_collect_package_target(${PackageProp} ${Tgt}) + endforeach() + + get_property(PackageSources GLOBAL PROPERTY YDB_CPP_${PackageProp}_SOURCES) + if (NOT PackageSources) + return() + endif() + + get_property(PackageInternalDeps GLOBAL PROPERTY YDB_CPP_${PackageProp}_INTERNAL_DEPS) + get_property(PackagePublicDeps GLOBAL PROPERTY YDB_CPP_${PackageProp}_PUBLIC_DEPS) + get_property(PackagePublicOpts GLOBAL PROPERTY YDB_CPP_${PackageProp}_PUBLIC_OPTS) + get_property(PackagePublicDefs GLOBAL PROPERTY YDB_CPP_${PackageProp}_PUBLIC_DEFS) + get_property(PackageIncludeDirs GLOBAL PROPERTY YDB_CPP_${PackageProp}_INCLUDE_DIRS) + + list(REMOVE_DUPLICATES PackageSources) + if (PackageInternalDeps) + list(REMOVE_DUPLICATES PackageInternalDeps) + endif() + if (PackagePublicDeps) + list(REMOVE_DUPLICATES PackagePublicDeps) + list(REMOVE_ITEM PackagePublicDeps ${PackageInternalDeps}) + endif() + if (PackagePublicOpts) + list(REMOVE_DUPLICATES PackagePublicOpts) + endif() + if (PackagePublicDefs) + list(REMOVE_DUPLICATES PackagePublicDefs) + endif() + if (PackageIncludeDirs) + list(REMOVE_DUPLICATES PackageIncludeDirs) + endif() + + foreach(src IN LISTS PackageSources) + if (NOT EXISTS "${src}") + set_source_files_properties("${src}" PROPERTIES GENERATED TRUE) + endif() + endforeach() + + add_library(${TargetName} STATIC ${PackageSources}) + target_include_directories(${TargetName} PUBLIC + $ + $ + $ + $ + $ + ) + target_include_directories(${TargetName} PRIVATE + ${PackageIncludeDirs} + ) + if (TargetName STREQUAL "libydb-cpp") + set_target_properties(${TargetName} PROPERTIES OUTPUT_NAME ydb-cpp) + endif() + target_compile_definitions(${TargetName} PUBLIC YDB_SDK_OSS ${PackagePublicDefs}) + target_link_libraries(${TargetName} PUBLIC ${PackagePublicDeps}) + if (PackageInternalDeps) + add_dependencies(${TargetName} ${PackageInternalDeps}) + endif() + if (PackagePublicOpts) + target_compile_options(${TargetName} PUBLIC ${PackagePublicOpts}) + endif() + _ydb_sdk_install_targets(TARGETS ${TargetName} COMPONENT ${ComponentName}) + endfunction() + + _ydb_sdk_create_package_library(CORE libydb-cpp libydb-cpp) + _ydb_sdk_create_package_library(LIBYDB_CPP_IAM ydb-cpp-iam libydb-cpp-iam) + _ydb_sdk_create_package_library(LIBYDB_CPP_OTEL_METRICS ydb-cpp-otel-metrics libydb-cpp-otel-metrics) + _ydb_sdk_create_package_library(LIBYDB_CPP_OTEL_TRACING ydb-cpp-otel-tracing libydb-cpp-otel-tracing) + _ydb_sdk_install_headers(${CMAKE_INSTALL_INCLUDEDIR}) install(EXPORT ydb-cpp-sdk-targets FILE ydb-cpp-sdk-targets.cmake diff --git a/cmake/PackSDK.cmake b/cmake/PackSDK.cmake index 9856d8e30aa..4d80d4f3918 100644 --- a/cmake/PackSDK.cmake +++ b/cmake/PackSDK.cmake @@ -19,6 +19,9 @@ set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_COMPONENTS_ALL libydb-cpp libydb-cpp-iam libydb-cpp-otel-metrics libydb-cpp-otel-tracing) set(CPACK_DEBIAN_LIBYDB_CPP_PACKAGE_NAME "libydb-cpp-dev") +set(CPACK_DEBIAN_LIBYDB_CPP_PACKAGE_DEPENDS + "libidn11-dev, libssl-dev, zlib1g-dev, libprotobuf-dev, libgrpc++-dev, libbrotli-dev, liblz4-dev, libzstd-dev, libbz2-dev, libxxhash-dev, libsnappy-dev, libdouble-conversion-dev, libgtest-dev, libre2-dev, libc-ares-dev, rapidjson-dev" +) set(CPACK_DEBIAN_LIBYDB_CPP_IAM_PACKAGE_NAME "libydb-cpp-iam-dev") set(CPACK_DEBIAN_LIBYDB_CPP_IAM_PACKAGE_DEPENDS "libydb-cpp-dev (= ${YDB_SDK_VERSION})") @@ -29,4 +32,11 @@ set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_METRICS_PACKAGE_DEPENDS "libydb-cpp-dev (= ${YD set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_TRACING_PACKAGE_NAME "libydb-cpp-otel-tracing-dev") set(CPACK_DEBIAN_LIBYDB_CPP_OTEL_TRACING_PACKAGE_DEPENDS "libydb-cpp-dev (= ${YDB_SDK_VERSION})") +foreach(component IN ITEMS libydb-cpp libydb-cpp-iam libydb-cpp-otel-metrics libydb-cpp-otel-tracing) + string(TOUPPER "${component}" component_upper) + string(REPLACE "-" "_" component_var "${component_upper}") + set("CPACK_DEBIAN_${component_upper}_PACKAGE_NAME" "${CPACK_DEBIAN_${component_var}_PACKAGE_NAME}") + set("CPACK_DEBIAN_${component_upper}_PACKAGE_DEPENDS" "${CPACK_DEBIAN_${component_var}_PACKAGE_DEPENDS}") +endforeach() + include(CPack) diff --git a/cmake/common.cmake b/cmake/common.cmake index 8a473b0ab06..be33436855d 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -185,9 +185,18 @@ function(_ydb_sdk_make_client_component CmpName Tgt) set(PKG_COMP_NAME "libydb-cpp") endif() - _ydb_sdk_install_targets(TARGETS ${Tgt} ${ARGN} COMPONENT ${PKG_COMP_NAME}) set(YDB-CPP-SDK_AVAILABLE_COMPONENTS ${YDB-CPP-SDK_AVAILABLE_COMPONENTS} ${CmpName} CACHE INTERNAL "") - set(YDB-CPP-SDK_COMPONENT_TARGETS ${YDB-CPP-SDK_COMPONENT_TARGETS} ${Tgt} CACHE INTERNAL "") + + if (PKG_COMP_NAME STREQUAL "libydb-cpp") + set(YDB_CPP_CORE_COMPONENT_TARGETS ${YDB_CPP_CORE_COMPONENT_TARGETS} ${Tgt} ${ARGN} CACHE INTERNAL "") + set(YDB-CPP-SDK_COMPONENT_TARGETS ${YDB-CPP-SDK_COMPONENT_TARGETS} libydb-cpp CACHE INTERNAL "") + else() + string(REPLACE "-" "_" PKG_PROP_NAME "${PKG_COMP_NAME}") + string(TOUPPER "${PKG_PROP_NAME}" PKG_PROP_NAME) + set(YDB_CPP_${PKG_PROP_NAME}_COMPONENT_TARGETS ${YDB_CPP_${PKG_PROP_NAME}_COMPONENT_TARGETS} ${Tgt} ${ARGN} CACHE INTERNAL "") + string(REGEX REPLACE "^lib" "" PKG_TARGET_NAME "${PKG_COMP_NAME}") + set(YDB-CPP-SDK_COMPONENT_TARGETS ${YDB-CPP-SDK_COMPONENT_TARGETS} ${PKG_TARGET_NAME} CACHE INTERNAL "") + endif() endfunction() function(_ydb_sdk_add_library Tgt) @@ -202,6 +211,7 @@ function(_ydb_sdk_add_library Tgt) set(libraryMode "INTERFACE") set(includeMode "INTERFACE") endif() + add_library(${Tgt} ${libraryMode}) target_include_directories(${Tgt} ${includeMode} $ @@ -211,6 +221,98 @@ function(_ydb_sdk_add_library Tgt) target_compile_definitions(${Tgt} ${includeMode} YDB_SDK_OSS ) + +endfunction() + +function(_ydb_sdk_collect_package_target PackageName Tgt) + if (NOT TARGET ${Tgt}) + return() + endif() + + get_target_property(aliased_target ${Tgt} ALIASED_TARGET) + if (aliased_target) + set(Tgt ${aliased_target}) + endif() + + get_target_property(imported_target ${Tgt} IMPORTED) + if (imported_target) + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_PUBLIC_DEPS ${Tgt}) + return() + endif() + + get_property(visited_targets GLOBAL PROPERTY YDB_CPP_${PackageName}_VISITED_TARGETS) + if (Tgt IN_LIST visited_targets) + return() + endif() + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_VISITED_TARGETS ${Tgt}) + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_INTERNAL_DEPS ${Tgt}) + + get_target_property(tgt_type ${Tgt} TYPE) + if (tgt_type STREQUAL "STATIC_LIBRARY" OR tgt_type STREQUAL "OBJECT_LIBRARY") + get_target_property(tgt_srcs ${Tgt} SOURCES) + get_target_property(tgt_source_dir ${Tgt} SOURCE_DIR) + if (tgt_srcs) + set(resolved_srcs "") + foreach(src IN LISTS tgt_srcs) + if (NOT src MATCHES "^\\$<") + if (NOT IS_ABSOLUTE "${src}") + set(src "${tgt_source_dir}/${src}") + endif() + list(APPEND resolved_srcs "${src}") + endif() + endforeach() + if (resolved_srcs) + get_target_property(tgt_compile_defs ${Tgt} COMPILE_DEFINITIONS) + get_target_property(tgt_compile_opts ${Tgt} COMPILE_OPTIONS) + foreach(resolved_src IN LISTS resolved_srcs) + if (tgt_compile_defs) + set_source_files_properties(${resolved_src} PROPERTIES COMPILE_DEFINITIONS "${tgt_compile_defs}") + endif() + if (tgt_compile_opts) + set_source_files_properties(${resolved_src} PROPERTIES COMPILE_OPTIONS "${tgt_compile_opts}") + endif() + endforeach() + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_SOURCES ${resolved_srcs}) + endif() + endif() + endif() + + foreach(prop IN ITEMS INCLUDE_DIRECTORIES INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(target_include_dirs ${Tgt} ${prop}) + if (target_include_dirs) + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_INCLUDE_DIRS ${target_include_dirs}) + endif() + endforeach() + + foreach(prop IN ITEMS LINK_LIBRARIES INTERFACE_LINK_LIBRARIES) + get_target_property(target_deps ${Tgt} ${prop}) + if (target_deps) + foreach(dep IN LISTS target_deps) + if (dep MATCHES "^\\$<") + continue() + endif() + if (TARGET ${dep}) + _ydb_sdk_collect_package_target(${PackageName} ${dep}) + else() + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_PUBLIC_DEPS ${dep}) + endif() + endforeach() + endif() + endforeach() + + get_target_property(tgt_pub_opts ${Tgt} INTERFACE_COMPILE_OPTIONS) + if (tgt_pub_opts) + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_PUBLIC_OPTS ${tgt_pub_opts}) + endif() + + get_target_property(tgt_pub_defs ${Tgt} INTERFACE_COMPILE_DEFINITIONS) + if (tgt_pub_defs) + set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_PUBLIC_DEFS ${tgt_pub_defs}) + endif() +endfunction() + +function(_ydb_sdk_collect_core_target Tgt) + _ydb_sdk_collect_package_target(CORE ${Tgt}) endfunction() function(_ydb_sdk_validate_public_headers) diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 4560fd662b3..c86f8617a9f 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -9,9 +9,32 @@ find_package(ZSTD REQUIRED) find_package(BZip2 REQUIRED) find_package(LZ4 REQUIRED) find_package(Snappy 1.1.8 REQUIRED) -find_package(base64 REQUIRED) +find_package(base64 QUIET) +if (NOT base64_FOUND) + include(FetchContent) + FetchContent_Declare( + base64 + GIT_REPOSITORY https://github.com/aklomp/base64.git + GIT_TAG v0.5.2 + ) + FetchContent_MakeAvailable(base64) + if (TARGET base64 AND NOT TARGET aklomp::base64) + add_library(aklomp::base64 ALIAS base64) + endif() +endif() find_package(Brotli 1.1.0 REQUIRED) -find_package(jwt-cpp REQUIRED) +find_package(jwt-cpp QUIET) +if (NOT jwt-cpp_FOUND) + include(FetchContent) + FetchContent_Declare( + jwt-cpp + GIT_REPOSITORY https://github.com/Thalhammer/jwt-cpp.git + GIT_TAG v0.6.0 + ) + set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(jwt-cpp) +endif() find_package(double-conversion REQUIRED) # OpenTelemetry diff --git a/cmake/global_flags.cmake b/cmake/global_flags.cmake index 23837966df6..6cee71ada62 100644 --- a/cmake/global_flags.cmake +++ b/cmake/global_flags.cmake @@ -33,7 +33,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux") # See: https://maskray.me/blog/2021-11-07-init-ctors-init-array string(APPEND COMMON_C_CXX_FLAGS " -fuse-init-array") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - string(APPEND COMMON_C_CXX_FLAGS " -fpermissive") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpermissive") endif() endif() diff --git a/cmake/install.cmake b/cmake/install.cmake index f56b57ea678..be2742b9e6a 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -20,6 +20,12 @@ function(_ydb_sdk_install_targets) return() endif() endforeach() + if (ARG_COMPONENT STREQUAL "libydb-cpp") + list(FILTER ARG_TARGETS INCLUDE REGEX "^libydb-cpp$") + if (NOT ARG_TARGETS) + return() + endif() + endif() install(TARGETS ${ARG_TARGETS} EXPORT ydb-cpp-sdk-targets CONFIGURATIONS RELEASE diff --git a/debian/libydb-cpp-dev.install b/debian/libydb-cpp-dev.install new file mode 100644 index 00000000000..8a43091a4dc --- /dev/null +++ b/debian/libydb-cpp-dev.install @@ -0,0 +1,4 @@ +debian/tmp/usr/lib/*/libydb-cpp.a usr/lib/ +debian/tmp/usr/include/ydb-cpp-sdk usr/include/ +debian/tmp/usr/include/__ydb_sdk_special_headers usr/include/ +debian/tmp/usr/lib/*/cmake/ydb-cpp-sdk usr/lib/*/cmake/ diff --git a/debian/libydb-cpp-iam-dev.install b/debian/libydb-cpp-iam-dev.install new file mode 100644 index 00000000000..7f0e4ef8c49 --- /dev/null +++ b/debian/libydb-cpp-iam-dev.install @@ -0,0 +1 @@ +debian/tmp/usr/lib/*/libydb-cpp-iam.a usr/lib/ diff --git a/debian/libydb-cpp-otel-metrics-dev.install b/debian/libydb-cpp-otel-metrics-dev.install new file mode 100644 index 00000000000..e6a00e6eca3 --- /dev/null +++ b/debian/libydb-cpp-otel-metrics-dev.install @@ -0,0 +1,2 @@ +debian/tmp/usr/lib/*/libydb-cpp-otel-metrics.a usr/lib/ +debian/tmp/usr/include/ydb-cpp-sdk/open_telemetry/metrics.h usr/include/ydb-cpp-sdk/open_telemetry/ diff --git a/debian/libydb-cpp-otel-tracing-dev.install b/debian/libydb-cpp-otel-tracing-dev.install new file mode 100644 index 00000000000..6ccc28fd9c5 --- /dev/null +++ b/debian/libydb-cpp-otel-tracing-dev.install @@ -0,0 +1,2 @@ +debian/tmp/usr/lib/*/libydb-cpp-otel-tracing.a usr/lib/ +debian/tmp/usr/include/ydb-cpp-sdk/open_telemetry/trace.h usr/include/ydb-cpp-sdk/open_telemetry/ diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh new file mode 100755 index 00000000000..a728960ec62 --- /dev/null +++ b/scripts/generate-debian-directory.sh @@ -0,0 +1,35 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +SOURCE_DIR=$(realpath "$SCRIPT_DIR/..") + +cd "$SOURCE_DIR" + +VERSION=$(grep -E 'YDB_SDK_VERSION = "[0-9]+\.[0-9]+\.[0-9]+"' src/version.h | sed -E 's/.*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/') +if [ -z "$VERSION" ]; then + echo "Error: Could not extract YDB_SDK_VERSION from src/version.h" + exit 1 +fi + +GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") + +CHANGELOG_FILE="debian/changelog" + +if [ ! -f "$CHANGELOG_FILE" ]; then + echo "Error: debian/changelog not found." + exit 1 +fi + +DEB_VERSION="${VERSION}-1" + +cat < "$CHANGELOG_FILE" +ydb-cpp-sdk (${DEB_VERSION}) unstable; urgency=low + + * Automated package build + * Git commit: ${GIT_COMMIT} + + -- YDB Team $(date -R) +EOF + +echo "Generated debian directory for version ${DEB_VERSION}" \ No newline at end of file diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh index 5f6b3de13cc..09afed81777 100755 --- a/scripts/test_deb_packages.sh +++ b/scripts/test_deb_packages.sh @@ -20,6 +20,6 @@ docker run --rm --network host \ -v "$DEB_DIR:/deb_packages:ro" \ -v "$YDB_DEPS_DIR:/ydb_deps:ro" \ ydb-cpp-sdk-deb-test \ - bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/base64;/ydb_deps/brotli;/ydb_deps/jwt-cpp' && make && ./test_app" + bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/base64;/ydb_deps/brotli;/ydb_deps/jwt-cpp;/ydb_deps/opentelemetry-cpp-install;/usr/local' && make && ./test_app" echo "Test successful!" \ No newline at end of file diff --git a/scripts/test_dpkg_buildpackage.sh b/scripts/test_dpkg_buildpackage.sh new file mode 100755 index 00000000000..4cee4ca049c --- /dev/null +++ b/scripts/test_dpkg_buildpackage.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -e + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +SOURCE_DIR=$(realpath "$SCRIPT_DIR/..") + +echo "Building test Docker image for dpkg-buildpackage..." + +# We need a Dockerfile that has all the build dependencies for dpkg-buildpackage +DOCKERFILE=$(mktemp) +cat < "$DOCKERFILE" +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y \\ + build-essential \\ + cmake \\ + pkg-config \\ + git \\ + libidn11-dev \\ + libssl-dev \\ + zlib1g-dev \\ + libprotobuf-dev \\ + protobuf-compiler \\ + libgrpc++-dev \\ + protobuf-compiler-grpc \\ + libbrotli-dev \\ + liblz4-dev \\ + libzstd-dev \\ + libbz2-dev \\ + libxxhash-dev \\ + libsnappy-dev \\ + libdouble-conversion-dev \\ + libgtest-dev \\ + libre2-dev \\ + libc-ares-dev \\ + rapidjson-dev \\ + debhelper \\ + dpkg-dev \\ + python3 \\ + python3-six \\ + ragel \\ + yasm \\ + && rm -rf /var/lib/apt/lists/* + +RUN git clone -b v1.12.0 https://github.com/open-telemetry/opentelemetry-cpp.git /tmp/otel && \\ + cd /tmp/otel && \\ + mkdir build && cd build && \\ + cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DWITH_OTLP_HTTP=OFF -DWITH_OTLP_GRPC=OFF -DWITH_PROMETHEUS=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \\ + make -j\$(nproc) && \\ + make install && \\ + rm -rf /tmp/otel + +WORKDIR /source +EOF + +docker build --network host -t ydb-cpp-sdk-dpkg-test -f "$DOCKERFILE" . +rm -f "$DOCKERFILE" + +# Prepare a clean tarball or just copy the directory inside the container +echo "Running dpkg-buildpackage in the container..." +docker run --rm --network host \ + -v "$SOURCE_DIR:/source:ro" \ + ydb-cpp-sdk-dpkg-test \ + bash -c "cp -r /source /tmp/source && cd /tmp/source && ./scripts/generate-debian-directory.sh && CMAKE_PREFIX_PATH='/usr/local' dpkg-buildpackage -us -uc -b -d -j\$(nproc)" + +echo "Test successful! dpkg-buildpackage works." \ No newline at end of file From 634d24320fa2260d553d0eaf6ec26ddfce41b7d5 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Mon, 18 May 2026 13:55:53 +0300 Subject: [PATCH 03/13] update readme accordingly --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index e53fb359d9c..3ae363d2d0d 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,46 @@ cmake --preset $sdk_configure_preset cmake --build --preset $sdk_configure_preset ``` +### Build `.deb` packages + +The SDK can be packaged as Debian development packages with CPack. The packaging build uses static libraries and produces the following packages: + +- `libydb-cpp-dev` — core SDK static library, public headers and CMake package files; +- `libydb-cpp-iam-dev` — IAM credentials plugin; +- `libydb-cpp-otel-metrics-dev` — OpenTelemetry metrics plugin; +- `libydb-cpp-otel-tracing-dev` — OpenTelemetry tracing plugin. + +The Debian packaging flow is intended for Ubuntu 24.04. Install the regular build dependencies first. If OpenTelemetry plugins are enabled, install `opentelemetry-cpp` as a CMake package as well; the default `package-deb-clang` preset expects it in `~/ydb_deps/opentelemetry-cpp-install`. + +To build `.deb` packages directly with CPack: + +```bash +cmake --preset package-deb-clang +cmake --build build-deb --target package -j$(nproc) +``` + +The generated `.deb` files are placed into `build-deb/`. + +To prepare Debian source-package metadata, run: + +```bash +./scripts/generate-debian-directory.sh +``` + +This updates `debian/changelog` from `src/version.h` and the current Git commit. The `debian/` directory contains package metadata and install manifests for `dpkg-buildpackage`/PPA builds. + +To validate a binary Debian package build in an Ubuntu 24.04 Docker container: + +```bash +./scripts/test_dpkg_buildpackage.sh +``` + +To smoke-test generated `.deb` packages with the sample consumer project: + +```bash +./scripts/test_deb_packages.sh build-deb +``` + ### Test Specify a level of parallelism by passing the `-j` option into the command below (e.g. `-j$(nproc)`) From 312af6a3fffb3727ed0439e71f5d145d09b6427f Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Mon, 18 May 2026 14:13:39 +0300 Subject: [PATCH 04/13] add .deb tests --- .github/workflows/tests.yaml | 97 +++++++++++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4eb73216f68..369e07c164f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -177,4 +177,99 @@ jobs: if [ -z "$(ls -A ./_install)" ]; then echo "Error: Installation directory is empty!" exit 1 - fi \ No newline at end of file + fi + + deb-packages: + name: "Test Debian Packages" + concurrency: + group: deb-packages-${{ github.ref }} + cancel-in-progress: true + runs-on: ubuntu-latest + steps: + - name: Checkout PR + uses: actions/checkout@v4 + if: github.event.pull_request.head.sha != '' + with: + submodules: true + ref: ${{ github.event.pull_request.head.sha }} + + - name: Checkout + uses: actions/checkout@v4 + if: github.event.pull_request.head.sha == '' + with: + submodules: true + + - name: Validate dpkg-buildpackage + shell: bash + run: | + ./scripts/test_dpkg_buildpackage.sh + + - name: Build CPack .deb packages + shell: bash + run: | + docker run --rm --network host \ + -v "$PWD:/source" \ + ubuntu:24.04 \ + bash -c ' + set -e + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + git \ + libidn11-dev \ + libssl-dev \ + zlib1g-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libgrpc++-dev \ + protobuf-compiler-grpc \ + libbrotli-dev \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + libxxhash-dev \ + libsnappy-dev \ + libdouble-conversion-dev \ + libgtest-dev \ + libre2-dev \ + libc-ares-dev \ + rapidjson-dev \ + python3 \ + python3-six \ + ragel \ + yasm + + git clone -b v1.12.0 https://github.com/open-telemetry/opentelemetry-cpp.git /tmp/otel + cmake -S /tmp/otel -B /tmp/otel/build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DWITH_OTLP_HTTP=OFF \ + -DWITH_OTLP_GRPC=OFF \ + -DWITH_PROMETHEUS=OFF \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local + cmake --build /tmp/otel/build -j$(nproc) + cmake --install /tmp/otel/build + + cd /source + ./scripts/generate-debian-directory.sh + cmake -S . -B build-deb \ + -DCMAKE_BUILD_TYPE=Release \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ + -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_PREFIX_PATH=/usr/local + cmake --build build-deb --target package -j$(nproc) + ' + + - name: Smoke-test generated .deb packages + shell: bash + run: | + ./scripts/test_deb_packages.sh build-deb \ No newline at end of file From 36641f538647fa4701bc882b9677a9f243bc4832 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Mon, 18 May 2026 15:35:12 +0300 Subject: [PATCH 05/13] Use system googleapis and generate debian dir from script --- .github/workflows/tests.yaml | 9 +- .gitignore | 1 + cmake/external_libs.cmake | 7 +- debian/libydb-cpp-dev.install | 4 - debian/libydb-cpp-iam-dev.install | 1 - debian/libydb-cpp-otel-metrics-dev.install | 2 - debian/libydb-cpp-otel-tracing-dev.install | 2 - scripts/generate-debian-directory.sh | 137 +++++++++++++++++- scripts/googleapis_deb/CMakeLists.txt | 86 +++++++++++ ...oogleapis-api-common-protosConfig.cmake.in | 6 + scripts/test_deb_packages.sh | 2 +- scripts/test_dpkg_buildpackage.sh | 8 +- tests/deb_package/test_project/main.cpp | 1 + 13 files changed, 245 insertions(+), 21 deletions(-) delete mode 100644 debian/libydb-cpp-dev.install delete mode 100644 debian/libydb-cpp-iam-dev.install delete mode 100644 debian/libydb-cpp-otel-metrics-dev.install delete mode 100644 debian/libydb-cpp-otel-tracing-dev.install create mode 100644 scripts/googleapis_deb/CMakeLists.txt create mode 100644 scripts/googleapis_deb/yandex-googleapis-api-common-protosConfig.cmake.in diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 369e07c164f..65e82443415 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -256,6 +256,11 @@ jobs: cmake --install /tmp/otel/build cd /source + cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex + cmake --build build_googleapis_deb -j$(nproc) + cmake --build build_googleapis_deb --target package + dpkg -i build_googleapis_deb/*.deb + ./scripts/generate-debian-directory.sh cmake -S . -B build-deb \ -DCMAKE_BUILD_TYPE=Release \ @@ -265,7 +270,9 @@ jobs: -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_PREFIX_PATH=/usr/local + -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + -DCMAKE_PREFIX_PATH="/usr/local;/usr/share/yandex" cmake --build build-deb --target package -j$(nproc) ' diff --git a/.gitignore b/.gitignore index dbe9ebc55e5..cf184074ffb 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,4 @@ CMakeUserPresets.json # Dev Containers !Dockerfile +debian/ diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index c86f8617a9f..bf21cd3e58a 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -54,7 +54,12 @@ if (YDB_SDK_USE_RAPID_JSON) endif() # api-common-protos -if (YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET) +option(YDB_SDK_USE_SYSTEM_GOOGLEAPIS "Use system-provided yandex-googleapis-api-common-protos" OFF) + +if (YDB_SDK_USE_SYSTEM_GOOGLEAPIS) + find_package(yandex-googleapis-api-common-protos REQUIRED) + add_library(api-common-protos ALIAS yandex-googleapis-api-common-protos::api-common-protos) +elseif (YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET) add_library(api-common-protos ALIAS ${YDB_SDK_GOOGLE_COMMON_PROTOS_TARGET}) else() file(MAKE_DIRECTORY ${YDB_SDK_BINARY_DIR}/third_party/api-common-protos) diff --git a/debian/libydb-cpp-dev.install b/debian/libydb-cpp-dev.install deleted file mode 100644 index 8a43091a4dc..00000000000 --- a/debian/libydb-cpp-dev.install +++ /dev/null @@ -1,4 +0,0 @@ -debian/tmp/usr/lib/*/libydb-cpp.a usr/lib/ -debian/tmp/usr/include/ydb-cpp-sdk usr/include/ -debian/tmp/usr/include/__ydb_sdk_special_headers usr/include/ -debian/tmp/usr/lib/*/cmake/ydb-cpp-sdk usr/lib/*/cmake/ diff --git a/debian/libydb-cpp-iam-dev.install b/debian/libydb-cpp-iam-dev.install deleted file mode 100644 index 7f0e4ef8c49..00000000000 --- a/debian/libydb-cpp-iam-dev.install +++ /dev/null @@ -1 +0,0 @@ -debian/tmp/usr/lib/*/libydb-cpp-iam.a usr/lib/ diff --git a/debian/libydb-cpp-otel-metrics-dev.install b/debian/libydb-cpp-otel-metrics-dev.install deleted file mode 100644 index e6a00e6eca3..00000000000 --- a/debian/libydb-cpp-otel-metrics-dev.install +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/lib/*/libydb-cpp-otel-metrics.a usr/lib/ -debian/tmp/usr/include/ydb-cpp-sdk/open_telemetry/metrics.h usr/include/ydb-cpp-sdk/open_telemetry/ diff --git a/debian/libydb-cpp-otel-tracing-dev.install b/debian/libydb-cpp-otel-tracing-dev.install deleted file mode 100644 index 6ccc28fd9c5..00000000000 --- a/debian/libydb-cpp-otel-tracing-dev.install +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/lib/*/libydb-cpp-otel-tracing.a usr/lib/ -debian/tmp/usr/include/ydb-cpp-sdk/open_telemetry/trace.h usr/include/ydb-cpp-sdk/open_telemetry/ diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index a728960ec62..4e67ed29bb5 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -14,22 +14,143 @@ fi GIT_COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown") -CHANGELOG_FILE="debian/changelog" +mkdir -p debian/ -if [ ! -f "$CHANGELOG_FILE" ]; then - echo "Error: debian/changelog not found." - exit 1 -fi +CHANGELOG_FILE="debian/changelog" DEB_VERSION="${VERSION}-1" -cat < "$CHANGELOG_FILE" +cat < "$CHANGELOG_FILE" ydb-cpp-sdk (${DEB_VERSION}) unstable; urgency=low * Automated package build * Git commit: ${GIT_COMMIT} -- YDB Team $(date -R) -EOF +EOF_CHANGELOG + +cat < debian/control +Source: ydb-cpp-sdk +Section: libdevel +Priority: optional +Maintainer: YDB Team +Build-Depends: debhelper-compat (= 13), + cmake, + pkg-config, + python3, + python3-six, + ragel, + yasm, + libidn11-dev, + libssl-dev, + zlib1g-dev, + libprotobuf-dev, + protobuf-compiler, + libgrpc++-dev, + protobuf-compiler-grpc, + libbrotli-dev, + liblz4-dev, + libzstd-dev, + libbz2-dev, + libxxhash-dev, + libsnappy-dev, + libdouble-conversion-dev, + libgtest-dev, + libre2-dev, + libc-ares-dev, + rapidjson-dev, + yandex-googleapis-api-common-protos +Standards-Version: 4.6.2 +Homepage: https://ydb.tech +Rules-Requires-Root: no + +Package: libydb-cpp-dev +Architecture: any +Depends: \${misc:Depends}, + libidn11-dev, + libssl-dev, + zlib1g-dev, + libprotobuf-dev, + libgrpc++-dev, + libbrotli-dev, + liblz4-dev, + libzstd-dev, + libbz2-dev, + libxxhash-dev, + libsnappy-dev, + libdouble-conversion-dev, + libgtest-dev, + libre2-dev, + libc-ares-dev, + rapidjson-dev, + yandex-googleapis-api-common-protos +Description: YDB C++ SDK development files + Static library, public headers and CMake package files for YDB C++ SDK. + +Package: libydb-cpp-iam-dev +Architecture: any +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) +Description: YDB C++ SDK IAM plugin development files + Static library for YDB C++ SDK IAM credentials plugin. + +Package: libydb-cpp-otel-metrics-dev +Architecture: any +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) +Description: YDB C++ SDK OpenTelemetry metrics plugin development files + Static library and headers for YDB C++ SDK OpenTelemetry metrics plugin. + +Package: libydb-cpp-otel-tracing-dev +Architecture: any +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) +Description: YDB C++ SDK OpenTelemetry tracing plugin development files + Static library and headers for YDB C++ SDK OpenTelemetry tracing plugin. +EOF_CONTROL + +cat <<'EOF_RULES' > debian/rules +#!/usr/bin/make -f + +export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +%: + dh $@ --buildsystem=cmake + +override_dh_auto_configure: + dh_auto_configure -- \ + -DCMAKE_BUILD_TYPE=Release \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ + -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + -DCMAKE_PREFIX_PATH=/usr/share/yandex +EOF_RULES +chmod +x debian/rules + +cat < debian/libydb-cpp-dev.install +debian/tmp/usr/share/yandex/lib/*/libydb-cpp.a usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/include/ydb-cpp-sdk usr/share/yandex/include/ +debian/tmp/usr/share/yandex/include/__ydb_sdk_special_headers usr/share/yandex/include/ +debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk usr/share/yandex/lib/*/cmake/ +EOF_INSTALL + +cat < debian/libydb-cpp-iam-dev.install +debian/tmp/usr/share/yandex/lib/*/libydb-cpp-iam.a usr/share/yandex/lib/ +EOF_INSTALL + +cat < debian/libydb-cpp-otel-metrics-dev.install +debian/tmp/usr/share/yandex/lib/*/libydb-cpp-otel-metrics.a usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/metrics.h usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/ +EOF_INSTALL + +cat < debian/libydb-cpp-otel-tracing-dev.install +debian/tmp/usr/share/yandex/lib/*/libydb-cpp-otel-tracing.a usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/trace.h usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/ +EOF_INSTALL + +mkdir -p debian/source +echo "3.0 (quilt)" > debian/source/format -echo "Generated debian directory for version ${DEB_VERSION}" \ No newline at end of file +echo "Generated debian directory for version ${DEB_VERSION}" diff --git a/scripts/googleapis_deb/CMakeLists.txt b/scripts/googleapis_deb/CMakeLists.txt new file mode 100644 index 00000000000..9c68cf40e1c --- /dev/null +++ b/scripts/googleapis_deb/CMakeLists.txt @@ -0,0 +1,86 @@ +cmake_minimum_required(VERSION 3.10) +project(yandex-googleapis-api-common-protos) + +set(CMAKE_CXX_STANDARD 20) + +find_package(Protobuf REQUIRED) + +set(PROTOS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/api-common-protos") + +file(GLOB_RECURSE PROTO_FILES "${PROTOS_DIR}/google/*.proto") + +# We need to generate the cpp files +set(PROTO_SRCS) +set(PROTO_HDRS) + +foreach(proto_file ${PROTO_FILES}) + file(RELATIVE_PATH rel_path "${PROTOS_DIR}" "${proto_file}") + string(REPLACE ".proto" ".pb.cc" cpp_file "${CMAKE_CURRENT_BINARY_DIR}/${rel_path}") + string(REPLACE ".proto" ".pb.h" h_file "${CMAKE_CURRENT_BINARY_DIR}/${rel_path}") + + get_filename_component(out_dir "${cpp_file}" DIRECTORY) + file(MAKE_DIRECTORY "${out_dir}") + + add_custom_command( + OUTPUT "${cpp_file}" "${h_file}" + COMMAND protobuf::protoc + ARGS --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" -I"${PROTOS_DIR}" "${proto_file}" + DEPENDS "${proto_file}" + COMMENT "Running C++ protocol buffer compiler on ${proto_file}" + VERBATIM + ) + + list(APPEND PROTO_SRCS "${cpp_file}") + list(APPEND PROTO_HDRS "${h_file}") +endforeach() + +add_library(api-common-protos STATIC ${PROTO_SRCS} ${PROTO_HDRS}) +add_library(yandex-googleapis-api-common-protos::api-common-protos ALIAS api-common-protos) + +target_include_directories(api-common-protos PUBLIC + $ + $ +) + +target_link_libraries(api-common-protos PUBLIC protobuf::libprotobuf) + +include(GNUInstallDirs) + +install(TARGETS api-common-protos + EXPORT yandex-googleapis-api-common-protos-targets + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) + +install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/google" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} + FILES_MATCHING PATTERN "*.pb.h" +) + +install(EXPORT yandex-googleapis-api-common-protos-targets + FILE yandex-googleapis-api-common-protosTargets.cmake + NAMESPACE yandex-googleapis-api-common-protos:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/yandex-googleapis-api-common-protos +) + +include(CMakePackageConfigHelpers) +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/yandex-googleapis-api-common-protosConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/yandex-googleapis-api-common-protosConfig.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/yandex-googleapis-api-common-protos +) + +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/yandex-googleapis-api-common-protosConfig.cmake" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/yandex-googleapis-api-common-protos +) + +set(CPACK_PACKAGE_NAME "yandex-googleapis-api-common-protos") +set(CPACK_PACKAGE_VERSION "1.0.0") +set(CPACK_GENERATOR "DEB") +set(CPACK_DEBIAN_PACKAGE_MAINTAINER "YDB Team ") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "libprotobuf-dev") +set(CPACK_PACKAGING_INSTALL_PREFIX "/usr/share/yandex") + +include(CPack) diff --git a/scripts/googleapis_deb/yandex-googleapis-api-common-protosConfig.cmake.in b/scripts/googleapis_deb/yandex-googleapis-api-common-protosConfig.cmake.in new file mode 100644 index 00000000000..9437a3ae170 --- /dev/null +++ b/scripts/googleapis_deb/yandex-googleapis-api-common-protosConfig.cmake.in @@ -0,0 +1,6 @@ +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +find_dependency(Protobuf) + +include("${CMAKE_CURRENT_LIST_DIR}/yandex-googleapis-api-common-protosTargets.cmake") diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh index 09afed81777..a40dcc52446 100755 --- a/scripts/test_deb_packages.sh +++ b/scripts/test_deb_packages.sh @@ -20,6 +20,6 @@ docker run --rm --network host \ -v "$DEB_DIR:/deb_packages:ro" \ -v "$YDB_DEPS_DIR:/ydb_deps:ro" \ ydb-cpp-sdk-deb-test \ - bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/base64;/ydb_deps/brotli;/ydb_deps/jwt-cpp;/ydb_deps/opentelemetry-cpp-install;/usr/local' && make && ./test_app" + bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/base64;/ydb_deps/brotli;/ydb_deps/jwt-cpp;/ydb_deps/opentelemetry-cpp-install;/usr/local;/usr/share/yandex' && make && ./test_app" echo "Test successful!" \ No newline at end of file diff --git a/scripts/test_dpkg_buildpackage.sh b/scripts/test_dpkg_buildpackage.sh index 4cee4ca049c..7b7822c9f50 100755 --- a/scripts/test_dpkg_buildpackage.sh +++ b/scripts/test_dpkg_buildpackage.sh @@ -61,6 +61,12 @@ echo "Running dpkg-buildpackage in the container..." docker run --rm --network host \ -v "$SOURCE_DIR:/source:ro" \ ydb-cpp-sdk-dpkg-test \ - bash -c "cp -r /source /tmp/source && cd /tmp/source && ./scripts/generate-debian-directory.sh && CMAKE_PREFIX_PATH='/usr/local' dpkg-buildpackage -us -uc -b -d -j\$(nproc)" + bash -c "cp -r /source /tmp/source && cd /tmp/source && \ + cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex && \ + cmake --build build_googleapis_deb -j\$(nproc) && \ + cmake --build build_googleapis_deb --target package && \ + dpkg -i build_googleapis_deb/*.deb && \ + ./scripts/generate-debian-directory.sh && \ + CMAKE_PREFIX_PATH='/usr/local;/usr/share/yandex' dpkg-buildpackage -us -uc -b -d -j\$(nproc)" echo "Test successful! dpkg-buildpackage works." \ No newline at end of file diff --git a/tests/deb_package/test_project/main.cpp b/tests/deb_package/test_project/main.cpp index afb83a28748..c4e05b3ca48 100644 --- a/tests/deb_package/test_project/main.cpp +++ b/tests/deb_package/test_project/main.cpp @@ -1,5 +1,6 @@ #include #include +#include #include From af049cc583d9fc1893e3629c7a8c32575d94a9a8 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Mon, 18 May 2026 17:27:00 +0300 Subject: [PATCH 06/13] fix protoc failures --- scripts/googleapis_deb/CMakeLists.txt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/googleapis_deb/CMakeLists.txt b/scripts/googleapis_deb/CMakeLists.txt index 9c68cf40e1c..0d8bc0623ef 100644 --- a/scripts/googleapis_deb/CMakeLists.txt +++ b/scripts/googleapis_deb/CMakeLists.txt @@ -6,6 +6,13 @@ set(CMAKE_CXX_STANDARD 20) find_package(Protobuf REQUIRED) set(PROTOS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/api-common-protos") +get_filename_component(PROTOS_DIR "${PROTOS_DIR}" ABSOLUTE) + +if(NOT EXISTS "${PROTOS_DIR}/google") + message(FATAL_ERROR + "api-common-protos sources not found under \"${PROTOS_DIR}/google\". " + "Initialize git submodules / third_party checkout.") +endif() file(GLOB_RECURSE PROTO_FILES "${PROTOS_DIR}/google/*.proto") @@ -23,10 +30,14 @@ foreach(proto_file ${PROTO_FILES}) add_custom_command( OUTPUT "${cpp_file}" "${h_file}" - COMMAND protobuf::protoc - ARGS --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" -I"${PROTOS_DIR}" "${proto_file}" + COMMAND "${Protobuf_PROTOC_EXECUTABLE}" + ARGS + "--cpp_out=${CMAKE_CURRENT_BINARY_DIR}" + "-I." + "${rel_path}" + WORKING_DIRECTORY "${PROTOS_DIR}" DEPENDS "${proto_file}" - COMMENT "Running C++ protocol buffer compiler on ${proto_file}" + COMMENT "Running C++ protocol buffer compiler on ${rel_path}" VERBATIM ) From 02d18f34ca21789b00c297c9bf17b611ffea062e Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Mon, 18 May 2026 19:53:47 +0300 Subject: [PATCH 07/13] fix tests --- .github/actions/prepare_vm/action.yaml | 22 +-------- .gitmodules | 6 +++ CMakePresets.json | 4 +- cmake/common.cmake | 60 +++++++++++++++++++++++- cmake/external_libs.cmake | 65 +++++++++++++++++++------- scripts/generate-debian-directory.sh | 2 +- scripts/test_deb_packages.sh | 2 +- scripts/test_dpkg_buildpackage.sh | 1 + third_party/aklomp-base64 | 1 + third_party/jwt-cpp | 1 + 10 files changed, 121 insertions(+), 43 deletions(-) create mode 160000 third_party/aklomp-base64 create mode 160000 third_party/jwt-cpp diff --git a/.github/actions/prepare_vm/action.yaml b/.github/actions/prepare_vm/action.yaml index d95ab630e67..a953ed84387 100644 --- a/.github/actions/prepare_vm/action.yaml +++ b/.github/actions/prepare_vm/action.yaml @@ -67,15 +67,6 @@ runs: cmake --install . --config Release --prefix ~/ydb_deps/grpc cd ../../ - # Install base64 - wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz - tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2 - mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - cmake --install . --config Release --prefix ~/ydb_deps/base64 - cd ../../ - # Install brotli wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0 @@ -85,17 +76,8 @@ runs: cmake --install . --config Release --prefix ~/ydb_deps/brotli cd ../../ - # Install jwt-cpp - wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz - tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0 - mkdir build && cd build - cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp - cd ../../ - # Clean up ccache -s sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \ - base64-0.5.2.tar.gz brotli-1.1.0.tar.gz jwt-cpp-0.7.0.tar.gz abseil-cpp-20230802.0 \ - protobuf-3.21.12 grpc-1.54.3 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0 + brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \ + protobuf-3.21.12 grpc-1.54.3 brotli-1.1.0 diff --git a/.gitmodules b/.gitmodules index ddeadcf3e65..e3048cb1508 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,9 @@ [submodule "third_party/FastLZ"] path = third_party/FastLZ url = https://github.com/ariya/FastLZ +[submodule "third_party/aklomp-base64"] + path = third_party/aklomp-base64 + url = https://github.com/aklomp/base64.git +[submodule "third_party/jwt-cpp"] + path = third_party/jwt-cpp + url = https://github.com/Thalhammer/jwt-cpp.git diff --git a/CMakePresets.json b/CMakePresets.json index 4186a63ab6b..36cc5b8413d 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -11,7 +11,7 @@ "displayName": "Generic Config", "cacheVariables": { "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", - "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/base64;~/ydb_deps/brotli;~/ydb_deps/jwt-cpp" + "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/brotli" }, "generator": "Ninja", "binaryDir": "${sourceDir}/build" @@ -78,7 +78,7 @@ "BUILD_SHARED_LIBS": "OFF", "CMAKE_EXE_LINKER_FLAGS": "", "CMAKE_SHARED_LINKER_FLAGS": "", - "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/base64;~/ydb_deps/brotli;~/ydb_deps/jwt-cpp;~/ydb_deps/opentelemetry-cpp-install" + "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/brotli;~/ydb_deps/opentelemetry-cpp-install" } }, { diff --git a/cmake/common.cmake b/cmake/common.cmake index be33436855d..fe2d5f30250 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -224,6 +224,41 @@ function(_ydb_sdk_add_library Tgt) endfunction() +function(_ydb_sdk_b64_set_codec_file BASE64_SRC_ROOT simd subdir) + set(_var "COMPILE_FLAGS_${simd}") + if (NOT DEFINED ${_var}) + return() + endif() + set(_flags "${${_var}}") + if (_flags STREQUAL " ") + return() + endif() + set(_codec_c "${BASE64_SRC_ROOT}/lib/arch/${subdir}/codec.c") + if (EXISTS "${_codec_c}") + set_source_files_properties("${_codec_c}" PROPERTIES COMPILE_FLAGS "${_flags}") + endif() +endfunction() + +# aklomp/base64 sets per-file COMPILE_FLAGS in its subdirectory; those properties are not +# visible when the same absolute paths are added to package static libs at the top level. +# Re-apply SIMD flags using the same compiler settings as third_party/aklomp-base64. +function(_ydb_sdk_apply_aklomp_base64_simd_file_flags BASE64_SRC_ROOT) + if (NOT IS_DIRECTORY "${BASE64_SRC_ROOT}") + return() + endif() + if (NOT EXISTS "${BASE64_SRC_ROOT}/cmake/Modules/TargetSIMDInstructionSet.cmake") + return() + endif() + include("${BASE64_SRC_ROOT}/cmake/Modules/TargetSIMDInstructionSet.cmake") + define_SIMD_compile_flags() + _ydb_sdk_b64_set_codec_file("${BASE64_SRC_ROOT}" SSSE3 ssse3) + _ydb_sdk_b64_set_codec_file("${BASE64_SRC_ROOT}" SSE41 sse41) + _ydb_sdk_b64_set_codec_file("${BASE64_SRC_ROOT}" SSE42 sse42) + _ydb_sdk_b64_set_codec_file("${BASE64_SRC_ROOT}" AVX avx) + _ydb_sdk_b64_set_codec_file("${BASE64_SRC_ROOT}" AVX2 avx2) + _ydb_sdk_b64_set_codec_file("${BASE64_SRC_ROOT}" AVX512 avx512) +endfunction() + function(_ydb_sdk_collect_package_target PackageName Tgt) if (NOT TARGET ${Tgt}) return() @@ -265,12 +300,35 @@ function(_ydb_sdk_collect_package_target PackageName Tgt) get_target_property(tgt_compile_defs ${Tgt} COMPILE_DEFINITIONS) get_target_property(tgt_compile_opts ${Tgt} COMPILE_OPTIONS) foreach(resolved_src IN LISTS resolved_srcs) + get_source_file_property(_file_simd_flags ${resolved_src} COMPILE_FLAGS) + if (_file_simd_flags STREQUAL "NOTFOUND") + set(_file_simd_flags "") + endif() + get_source_file_property(_file_defs ${resolved_src} COMPILE_DEFINITIONS) + if (_file_defs STREQUAL "NOTFOUND") + set(_file_defs "") + endif() + if (tgt_compile_defs) - set_source_files_properties(${resolved_src} PROPERTIES COMPILE_DEFINITIONS "${tgt_compile_defs}") + if (_file_defs) + set_source_files_properties(${resolved_src} PROPERTIES + COMPILE_DEFINITIONS "${_file_defs};${tgt_compile_defs}") + else() + set_source_files_properties(${resolved_src} PROPERTIES + COMPILE_DEFINITIONS "${tgt_compile_defs}") + endif() + elseif (_file_defs) + set_source_files_properties(${resolved_src} PROPERTIES + COMPILE_DEFINITIONS "${_file_defs}") endif() + if (tgt_compile_opts) set_source_files_properties(${resolved_src} PROPERTIES COMPILE_OPTIONS "${tgt_compile_opts}") endif() + + if (_file_simd_flags) + set_source_files_properties(${resolved_src} PROPERTIES COMPILE_FLAGS "${_file_simd_flags}") + endif() endforeach() set_property(GLOBAL APPEND PROPERTY YDB_CPP_${PackageName}_SOURCES ${resolved_srcs}) endif() diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index bf21cd3e58a..976f1185ba2 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -9,31 +9,60 @@ find_package(ZSTD REQUIRED) find_package(BZip2 REQUIRED) find_package(LZ4 REQUIRED) find_package(Snappy 1.1.8 REQUIRED) +set(_ydb_sdk_vendor_base64 "${YDB_SDK_SOURCE_DIR}/third_party/aklomp-base64") +set(_ydb_sdk_vendor_jwt_cpp "${YDB_SDK_SOURCE_DIR}/third_party/jwt-cpp") + find_package(base64 QUIET) if (NOT base64_FOUND) - include(FetchContent) - FetchContent_Declare( - base64 - GIT_REPOSITORY https://github.com/aklomp/base64.git - GIT_TAG v0.5.2 - ) - FetchContent_MakeAvailable(base64) - if (TARGET base64 AND NOT TARGET aklomp::base64) - add_library(aklomp::base64 ALIAS base64) + if (EXISTS "${_ydb_sdk_vendor_base64}/CMakeLists.txt") + set(BASE64_WERROR OFF CACHE BOOL "" FORCE) + set(BASE64_BUILD_CLI OFF CACHE BOOL "" FORCE) + add_subdirectory("${_ydb_sdk_vendor_base64}" "${YDB_SDK_BINARY_DIR}/third_party/aklomp-base64") + _ydb_sdk_apply_aklomp_base64_simd_file_flags("${_ydb_sdk_vendor_base64}") + if (TARGET base64 AND NOT TARGET aklomp::base64) + add_library(aklomp::base64 ALIAS base64) + endif() + elseif (FETCHCONTENT_FULLY_DISCONNECTED) + message(FATAL_ERROR + "base64 is required but was not found (find_package), vendored sources are missing at " + "'${_ydb_sdk_vendor_base64}', and FETCHCONTENT_FULLY_DISCONNECTED=ON prevents FetchContent.") + else() + include(FetchContent) + FetchContent_Declare( + base64 + GIT_REPOSITORY https://github.com/aklomp/base64.git + GIT_TAG v0.5.2 + ) + FetchContent_MakeAvailable(base64) + get_target_property(_ydb_sdk_b64_src_dir base64 SOURCE_DIR) + _ydb_sdk_apply_aklomp_base64_simd_file_flags("${_ydb_sdk_b64_src_dir}") + if (TARGET base64 AND NOT TARGET aklomp::base64) + add_library(aklomp::base64 ALIAS base64) + endif() endif() endif() find_package(Brotli 1.1.0 REQUIRED) find_package(jwt-cpp QUIET) if (NOT jwt-cpp_FOUND) - include(FetchContent) - FetchContent_Declare( - jwt-cpp - GIT_REPOSITORY https://github.com/Thalhammer/jwt-cpp.git - GIT_TAG v0.6.0 - ) - set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) - set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) - FetchContent_MakeAvailable(jwt-cpp) + if (EXISTS "${_ydb_sdk_vendor_jwt_cpp}/CMakeLists.txt") + set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) + add_subdirectory("${_ydb_sdk_vendor_jwt_cpp}" "${YDB_SDK_BINARY_DIR}/third_party/jwt-cpp") + elseif (FETCHCONTENT_FULLY_DISCONNECTED) + message(FATAL_ERROR + "jwt-cpp is required but was not found (find_package), vendored sources are missing at " + "'${_ydb_sdk_vendor_jwt_cpp}', and FETCHCONTENT_FULLY_DISCONNECTED=ON prevents FetchContent.") + else() + include(FetchContent) + FetchContent_Declare( + jwt-cpp + GIT_REPOSITORY https://github.com/Thalhammer/jwt-cpp.git + GIT_TAG v0.6.0 + ) + set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(jwt-cpp) + endif() endif() find_package(double-conversion REQUIRED) diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index 4e67ed29bb5..503af4bba7e 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -125,7 +125,7 @@ override_dh_auto_configure: -DBUILD_SHARED_LIBS=OFF \ -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH=/usr/share/yandex + -DCMAKE_PREFIX_PATH="/usr/local;/usr/share/yandex" EOF_RULES chmod +x debian/rules diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh index a40dcc52446..e81c88be247 100755 --- a/scripts/test_deb_packages.sh +++ b/scripts/test_deb_packages.sh @@ -20,6 +20,6 @@ docker run --rm --network host \ -v "$DEB_DIR:/deb_packages:ro" \ -v "$YDB_DEPS_DIR:/ydb_deps:ro" \ ydb-cpp-sdk-deb-test \ - bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/base64;/ydb_deps/brotli;/ydb_deps/jwt-cpp;/ydb_deps/opentelemetry-cpp-install;/usr/local;/usr/share/yandex' && make && ./test_app" + bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/brotli;/ydb_deps/opentelemetry-cpp-install;/usr/local;/usr/share/yandex' && make && ./test_app" echo "Test successful!" \ No newline at end of file diff --git a/scripts/test_dpkg_buildpackage.sh b/scripts/test_dpkg_buildpackage.sh index 7b7822c9f50..725359d6ee3 100755 --- a/scripts/test_dpkg_buildpackage.sh +++ b/scripts/test_dpkg_buildpackage.sh @@ -62,6 +62,7 @@ docker run --rm --network host \ -v "$SOURCE_DIR:/source:ro" \ ydb-cpp-sdk-dpkg-test \ bash -c "cp -r /source /tmp/source && cd /tmp/source && \ + rm -rf build_googleapis_deb && \ cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex && \ cmake --build build_googleapis_deb -j\$(nproc) && \ cmake --build build_googleapis_deb --target package && \ diff --git a/third_party/aklomp-base64 b/third_party/aklomp-base64 new file mode 160000 index 00000000000..8bdda2d47ca --- /dev/null +++ b/third_party/aklomp-base64 @@ -0,0 +1 @@ +Subproject commit 8bdda2d47caf8b066999c5bd01069e55bcd0d396 diff --git a/third_party/jwt-cpp b/third_party/jwt-cpp new file mode 160000 index 00000000000..4a537e96989 --- /dev/null +++ b/third_party/jwt-cpp @@ -0,0 +1 @@ +Subproject commit 4a537e969891dde542ad8b1a4a214955a83be29f From 41a245b9412b97fc83bbc73a2a430aec20f7f893 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Tue, 19 May 2026 11:59:01 +0300 Subject: [PATCH 08/13] fix ci failures --- .github/workflows/tests.yaml | 1 + cmake/external_libs.cmake | 4 ++++ cmake/ydb-cpp-sdk-config.cmake.in | 4 ++++ scripts/generate-debian-directory.sh | 7 +++++++ scripts/test_deb_packages.sh | 5 +---- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 65e82443415..6d124ffb46e 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -274,6 +274,7 @@ jobs: -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ -DCMAKE_PREFIX_PATH="/usr/local;/usr/share/yandex" cmake --build build-deb --target package -j$(nproc) + cp build_googleapis_deb/*.deb build-deb/ ' - name: Smoke-test generated .deb packages diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 976f1185ba2..77935b1fc48 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -17,6 +17,7 @@ if (NOT base64_FOUND) if (EXISTS "${_ydb_sdk_vendor_base64}/CMakeLists.txt") set(BASE64_WERROR OFF CACHE BOOL "" FORCE) set(BASE64_BUILD_CLI OFF CACHE BOOL "" FORCE) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) add_subdirectory("${_ydb_sdk_vendor_base64}" "${YDB_SDK_BINARY_DIR}/third_party/aklomp-base64") _ydb_sdk_apply_aklomp_base64_simd_file_flags("${_ydb_sdk_vendor_base64}") if (TARGET base64 AND NOT TARGET aklomp::base64) @@ -33,6 +34,7 @@ if (NOT base64_FOUND) GIT_REPOSITORY https://github.com/aklomp/base64.git GIT_TAG v0.5.2 ) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) FetchContent_MakeAvailable(base64) get_target_property(_ydb_sdk_b64_src_dir base64 SOURCE_DIR) _ydb_sdk_apply_aklomp_base64_simd_file_flags("${_ydb_sdk_b64_src_dir}") @@ -47,6 +49,7 @@ if (NOT jwt-cpp_FOUND) if (EXISTS "${_ydb_sdk_vendor_jwt_cpp}/CMakeLists.txt") set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) add_subdirectory("${_ydb_sdk_vendor_jwt_cpp}" "${YDB_SDK_BINARY_DIR}/third_party/jwt-cpp") elseif (FETCHCONTENT_FULLY_DISCONNECTED) message(FATAL_ERROR @@ -61,6 +64,7 @@ if (NOT jwt-cpp_FOUND) ) set(JWT_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) set(JWT_BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME libydb-cpp) FetchContent_MakeAvailable(jwt-cpp) endif() endif() diff --git a/cmake/ydb-cpp-sdk-config.cmake.in b/cmake/ydb-cpp-sdk-config.cmake.in index 3a1c8299332..9ab167f4f53 100644 --- a/cmake/ydb-cpp-sdk-config.cmake.in +++ b/cmake/ydb-cpp-sdk-config.cmake.in @@ -26,6 +26,10 @@ find_package(jwt-cpp REQUIRED) find_package(GTest REQUIRED) find_package(double-conversion REQUIRED) +if (@YDB_SDK_USE_SYSTEM_GOOGLEAPIS@) + find_package(yandex-googleapis-api-common-protos REQUIRED) +endif() + if (@YDB_SDK_USE_RAPID_JSON@) find_package(RapidJSON REQUIRED) diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index 503af4bba7e..50eee915a0f 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -134,6 +134,12 @@ debian/tmp/usr/share/yandex/lib/*/libydb-cpp.a usr/share/yandex/lib/ debian/tmp/usr/share/yandex/include/ydb-cpp-sdk usr/share/yandex/include/ debian/tmp/usr/share/yandex/include/__ydb_sdk_special_headers usr/share/yandex/include/ debian/tmp/usr/share/yandex/lib/*/cmake/ydb-cpp-sdk usr/share/yandex/lib/*/cmake/ +debian/tmp/usr/share/yandex/include/libbase64.h usr/share/yandex/include/ +debian/tmp/usr/share/yandex/lib/*/libbase64.a usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/lib/*/cmake/base64 usr/share/yandex/lib/*/cmake/ +debian/tmp/usr/share/yandex/include/picojson usr/share/yandex/include/ +debian/tmp/usr/share/yandex/include/jwt-cpp usr/share/yandex/include/ +debian/tmp/usr/share/yandex/cmake/jwt-cpp* usr/share/yandex/cmake/ EOF_INSTALL cat < debian/libydb-cpp-iam-dev.install @@ -150,6 +156,7 @@ debian/tmp/usr/share/yandex/lib/*/libydb-cpp-otel-tracing.a usr/share/yandex/lib debian/tmp/usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/trace.h usr/share/yandex/include/ydb-cpp-sdk/open_telemetry/ EOF_INSTALL + mkdir -p debian/source echo "3.0 (quilt)" > debian/source/format diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh index e81c88be247..814c96bb7b3 100755 --- a/scripts/test_deb_packages.sh +++ b/scripts/test_deb_packages.sh @@ -13,13 +13,10 @@ TEST_DIR=$(realpath "$SCRIPT_DIR/../tests/deb_package") echo "Building test Docker image..." docker build --network host -t ydb-cpp-sdk-deb-test "$TEST_DIR" -YDB_DEPS_DIR=$(realpath "$SCRIPT_DIR/../../ydb_deps") - echo "Running test container..." docker run --rm --network host \ -v "$DEB_DIR:/deb_packages:ro" \ - -v "$YDB_DEPS_DIR:/ydb_deps:ro" \ ydb-cpp-sdk-deb-test \ - bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/ydb_deps/absl;/ydb_deps/protobuf;/ydb_deps/grpc;/ydb_deps/brotli;/ydb_deps/opentelemetry-cpp-install;/usr/local;/usr/share/yandex' && make && ./test_app" + bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/usr/local;/usr/share/yandex' && make && ./test_app" echo "Test successful!" \ No newline at end of file From ddd6b86aa99fea081daf389ce007c54536ac7cc9 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Tue, 19 May 2026 15:02:06 +0300 Subject: [PATCH 09/13] fix: add otel to build --- scripts/generate-debian-directory.sh | 20 +++++++++++++++++++- scripts/test_dpkg_buildpackage.sh | 11 +---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index 50eee915a0f..0a2f3ad1657 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -36,6 +36,7 @@ Priority: optional Maintainer: YDB Team Build-Depends: debhelper-compat (= 13), cmake, + git, pkg-config, python3, python3-six, @@ -110,11 +111,24 @@ cat <<'EOF_RULES' > debian/rules #!/usr/bin/make -f export DEB_BUILD_MAINT_OPTIONS = hardening=+all +OTEL_INSTALL_DIR := $(CURDIR)/debian/otel-install %: dh $@ --buildsystem=cmake override_dh_auto_configure: + git clone -b v1.12.0 --depth 1 https://github.com/open-telemetry/opentelemetry-cpp.git debian/otel-src + cmake -S debian/otel-src -B debian/otel-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTING=OFF \ + -DWITH_OTLP_HTTP=OFF \ + -DWITH_OTLP_GRPC=OFF \ + -DWITH_PROMETHEUS=OFF \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=$(OTEL_INSTALL_DIR) + cmake --build debian/otel-build -j$$(nproc) + cmake --install debian/otel-build dh_auto_configure -- \ -DCMAKE_BUILD_TYPE=Release \ -DYDB_SDK_INSTALL=ON \ @@ -125,7 +139,11 @@ override_dh_auto_configure: -DBUILD_SHARED_LIBS=OFF \ -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="/usr/local;/usr/share/yandex" + -DCMAKE_PREFIX_PATH="$(OTEL_INSTALL_DIR);/usr/share/yandex" + +override_dh_auto_clean: + dh_auto_clean + rm -rf debian/otel-src debian/otel-build debian/otel-install EOF_RULES chmod +x debian/rules diff --git a/scripts/test_dpkg_buildpackage.sh b/scripts/test_dpkg_buildpackage.sh index 725359d6ee3..bf5e53ab1d7 100755 --- a/scripts/test_dpkg_buildpackage.sh +++ b/scripts/test_dpkg_buildpackage.sh @@ -41,15 +41,6 @@ RUN apt-get update && apt-get install -y \\ ragel \\ yasm \\ && rm -rf /var/lib/apt/lists/* - -RUN git clone -b v1.12.0 https://github.com/open-telemetry/opentelemetry-cpp.git /tmp/otel && \\ - cd /tmp/otel && \\ - mkdir build && cd build && \\ - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DWITH_OTLP_HTTP=OFF -DWITH_OTLP_GRPC=OFF -DWITH_PROMETHEUS=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \\ - make -j\$(nproc) && \\ - make install && \\ - rm -rf /tmp/otel - WORKDIR /source EOF @@ -68,6 +59,6 @@ docker run --rm --network host \ cmake --build build_googleapis_deb --target package && \ dpkg -i build_googleapis_deb/*.deb && \ ./scripts/generate-debian-directory.sh && \ - CMAKE_PREFIX_PATH='/usr/local;/usr/share/yandex' dpkg-buildpackage -us -uc -b -d -j\$(nproc)" + dpkg-buildpackage -us -uc -b -d -j\$(nproc)" echo "Test successful! dpkg-buildpackage works." \ No newline at end of file From ba9a176bfba9df2b0a4ecaf3d627032b308019d2 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Tue, 19 May 2026 16:49:22 +0300 Subject: [PATCH 10/13] fix otel vendoring --- .github/workflows/tests.yaml | 15 +-------- .gitmodules | 3 ++ cmake/external_libs.cmake | 49 +++++++++++++++++++++++++++- scripts/generate-debian-directory.sh | 23 +++---------- scripts/test_deb_packages.sh | 2 +- tests/deb_package/Dockerfile | 8 ----- third_party/opentelemetry-cpp | 1 + 7 files changed, 59 insertions(+), 42 deletions(-) create mode 160000 third_party/opentelemetry-cpp diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6d124ffb46e..f0816775abb 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -242,19 +242,6 @@ jobs: ragel \ yasm - git clone -b v1.12.0 https://github.com/open-telemetry/opentelemetry-cpp.git /tmp/otel - cmake -S /tmp/otel -B /tmp/otel/build \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=OFF \ - -DWITH_OTLP_HTTP=OFF \ - -DWITH_OTLP_GRPC=OFF \ - -DWITH_PROMETHEUS=OFF \ - -DCMAKE_CXX_STANDARD=20 \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_INSTALL_PREFIX=/usr/local - cmake --build /tmp/otel/build -j$(nproc) - cmake --install /tmp/otel/build - cd /source cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex cmake --build build_googleapis_deb -j$(nproc) @@ -272,7 +259,7 @@ jobs: -DBUILD_SHARED_LIBS=OFF \ -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="/usr/local;/usr/share/yandex" + -DCMAKE_PREFIX_PATH="/usr/share/yandex" cmake --build build-deb --target package -j$(nproc) cp build_googleapis_deb/*.deb build-deb/ ' diff --git a/.gitmodules b/.gitmodules index e3048cb1508..d5951cffaab 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "third_party/jwt-cpp"] path = third_party/jwt-cpp url = https://github.com/Thalhammer/jwt-cpp.git +[submodule "third_party/opentelemetry-cpp"] + path = third_party/opentelemetry-cpp + url = https://github.com/open-telemetry/opentelemetry-cpp.git diff --git a/cmake/external_libs.cmake b/cmake/external_libs.cmake index 77935b1fc48..334dc480884 100644 --- a/cmake/external_libs.cmake +++ b/cmake/external_libs.cmake @@ -71,8 +71,55 @@ endif() find_package(double-conversion REQUIRED) # OpenTelemetry +set(_ydb_sdk_vendor_otel "${YDB_SDK_SOURCE_DIR}/third_party/opentelemetry-cpp") + if (YDB_SDK_ENABLE_OTEL_METRICS OR YDB_SDK_ENABLE_OTEL_TRACE) - find_package(opentelemetry-cpp REQUIRED) + find_package(opentelemetry-cpp QUIET) + if (NOT opentelemetry-cpp_FOUND) + set(BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(WITH_OTLP_HTTP OFF CACHE BOOL "" FORCE) + set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE) + set(WITH_PROMETHEUS OFF CACHE BOOL "" FORCE) + set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE) + set(OTELCPP_MAINTAINER_MODE OFF CACHE BOOL "" FORCE) + set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "" FORCE) + set(WITH_STL "CXX20" CACHE STRING "" FORCE) + set(OPENTELEMETRY_INSTALL ON CACHE BOOL "" FORCE) + if (EXISTS "${_ydb_sdk_vendor_otel}/CMakeLists.txt") + add_subdirectory("${_ydb_sdk_vendor_otel}" "${YDB_SDK_BINARY_DIR}/third_party/opentelemetry-cpp") + elseif (FETCHCONTENT_FULLY_DISCONNECTED) + message(FATAL_ERROR + "opentelemetry-cpp is required but was not found (find_package), vendored sources are missing at " + "'${_ydb_sdk_vendor_otel}', and FETCHCONTENT_FULLY_DISCONNECTED=ON prevents FetchContent.") + else() + include(FetchContent) + FetchContent_Declare( + opentelemetry-cpp + GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git + GIT_TAG v1.12.0 + ) + FetchContent_MakeAvailable(opentelemetry-cpp) + endif() + # Create namespaced aliases matching the installed opentelemetry-cpp::* targets + if (TARGET opentelemetry_api AND NOT TARGET opentelemetry-cpp::api) + add_library(opentelemetry-cpp::api ALIAS opentelemetry_api) + endif() + if (TARGET opentelemetry_metrics AND NOT TARGET opentelemetry-cpp::metrics) + add_library(opentelemetry-cpp::metrics ALIAS opentelemetry_metrics) + endif() + if (TARGET opentelemetry_trace AND NOT TARGET opentelemetry-cpp::trace) + add_library(opentelemetry-cpp::trace ALIAS opentelemetry_trace) + endif() + if (TARGET opentelemetry_common AND NOT TARGET opentelemetry-cpp::common) + add_library(opentelemetry-cpp::common ALIAS opentelemetry_common) + endif() + if (TARGET opentelemetry_resources AND NOT TARGET opentelemetry-cpp::resources) + add_library(opentelemetry-cpp::resources ALIAS opentelemetry_resources) + endif() + if (TARGET opentelemetry_version AND NOT TARGET opentelemetry-cpp::version) + add_library(opentelemetry-cpp::version ALIAS opentelemetry_version) + endif() + endif() endif() # RapidJSON diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index 0a2f3ad1657..ec3ee29de66 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -111,24 +111,11 @@ cat <<'EOF_RULES' > debian/rules #!/usr/bin/make -f export DEB_BUILD_MAINT_OPTIONS = hardening=+all -OTEL_INSTALL_DIR := $(CURDIR)/debian/otel-install %: dh $@ --buildsystem=cmake override_dh_auto_configure: - git clone -b v1.12.0 --depth 1 https://github.com/open-telemetry/opentelemetry-cpp.git debian/otel-src - cmake -S debian/otel-src -B debian/otel-build \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=OFF \ - -DWITH_OTLP_HTTP=OFF \ - -DWITH_OTLP_GRPC=OFF \ - -DWITH_PROMETHEUS=OFF \ - -DCMAKE_CXX_STANDARD=20 \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DCMAKE_INSTALL_PREFIX=$(OTEL_INSTALL_DIR) - cmake --build debian/otel-build -j$$(nproc) - cmake --install debian/otel-build dh_auto_configure -- \ -DCMAKE_BUILD_TYPE=Release \ -DYDB_SDK_INSTALL=ON \ @@ -139,11 +126,7 @@ override_dh_auto_configure: -DBUILD_SHARED_LIBS=OFF \ -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DCMAKE_PREFIX_PATH="$(OTEL_INSTALL_DIR);/usr/share/yandex" - -override_dh_auto_clean: - dh_auto_clean - rm -rf debian/otel-src debian/otel-build debian/otel-install + -DCMAKE_PREFIX_PATH="/usr/share/yandex" EOF_RULES chmod +x debian/rules @@ -158,6 +141,10 @@ debian/tmp/usr/share/yandex/lib/*/cmake/base64 usr/share/yandex/lib/*/cmake/ debian/tmp/usr/share/yandex/include/picojson usr/share/yandex/include/ debian/tmp/usr/share/yandex/include/jwt-cpp usr/share/yandex/include/ debian/tmp/usr/share/yandex/cmake/jwt-cpp* usr/share/yandex/cmake/ +debian/tmp/usr/share/yandex/include/opentelemetry usr/share/yandex/include/ +debian/tmp/usr/share/yandex/lib/*/libopentelemetry_* usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/lib/*/cmake/opentelemetry-cpp usr/share/yandex/lib/*/cmake/ +debian/tmp/usr/share/yandex/lib/*/pkgconfig/opentelemetry_*.pc usr/share/yandex/lib/*/pkgconfig/ EOF_INSTALL cat < debian/libydb-cpp-iam-dev.install diff --git a/scripts/test_deb_packages.sh b/scripts/test_deb_packages.sh index 814c96bb7b3..2f2d198c0f9 100755 --- a/scripts/test_deb_packages.sh +++ b/scripts/test_deb_packages.sh @@ -17,6 +17,6 @@ echo "Running test container..." docker run --rm --network host \ -v "$DEB_DIR:/deb_packages:ro" \ ydb-cpp-sdk-deb-test \ - bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/usr/local;/usr/share/yandex' && make && ./test_app" + bash -c "apt-get update && apt-get install -y /deb_packages/*.deb && mkdir build && cd build && cmake .. -DCMAKE_PREFIX_PATH='/usr/share/yandex' && make && ./test_app" echo "Test successful!" \ No newline at end of file diff --git a/tests/deb_package/Dockerfile b/tests/deb_package/Dockerfile index 3c78acbfc24..c6ed11a77b3 100644 --- a/tests/deb_package/Dockerfile +++ b/tests/deb_package/Dockerfile @@ -27,13 +27,5 @@ RUN apt-get update && apt-get install -y \ rapidjson-dev \ && rm -rf /var/lib/apt/lists/* -RUN git clone -b v1.12.0 https://github.com/open-telemetry/opentelemetry-cpp.git /tmp/otel && \ - cd /tmp/otel && \ - mkdir build && cd build && \ - cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -DWITH_OTLP_HTTP=OFF -DWITH_OTLP_GRPC=OFF -DWITH_PROMETHEUS=OFF -DCMAKE_CXX_STANDARD=20 -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \ - make -j$(nproc) && \ - make install && \ - rm -rf /tmp/otel - COPY test_project /test_project WORKDIR /test_project \ No newline at end of file diff --git a/third_party/opentelemetry-cpp b/third_party/opentelemetry-cpp new file mode 160000 index 00000000000..46e20a42aef --- /dev/null +++ b/third_party/opentelemetry-cpp @@ -0,0 +1 @@ +Subproject commit 46e20a42aef521b9cc1b7207310bfefcd490741a From 5e3ef7bd0b66fce62502c5a9850b06f89ae9b0cb Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Thu, 21 May 2026 10:24:03 +0300 Subject: [PATCH 11/13] add ppa upload script --- .github/workflows/ppa_publish.yaml | 121 +++++++ README.md | 51 +++ plans/ppa-setup.md | 275 ++++++++++++++++ plans/ppa-upload-plan.md | 266 ++++++++++++++++ scripts/generate-debian-directory.sh | 43 ++- .../generate-debian-directory.sh | 89 ++++++ scripts/otel_deb/CMakeLists.txt | 23 ++ scripts/otel_deb/generate-debian-directory.sh | 87 +++++ scripts/upload_to_ppa.sh | 300 ++++++++++++++++++ 9 files changed, 1243 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ppa_publish.yaml create mode 100644 plans/ppa-setup.md create mode 100644 plans/ppa-upload-plan.md create mode 100755 scripts/googleapis_deb/generate-debian-directory.sh create mode 100644 scripts/otel_deb/CMakeLists.txt create mode 100755 scripts/otel_deb/generate-debian-directory.sh create mode 100755 scripts/upload_to_ppa.sh diff --git a/.github/workflows/ppa_publish.yaml b/.github/workflows/ppa_publish.yaml new file mode 100644 index 00000000000..53fd813c280 --- /dev/null +++ b/.github/workflows/ppa_publish.yaml @@ -0,0 +1,121 @@ +name: Publish to PPA + +on: + release: + types: [published] + workflow_dispatch: + inputs: + series: + description: "Ubuntu series (e.g. noble, jammy)" + required: false + default: "noble" + packages: + description: "Which packages to upload" + required: false + default: "all" + type: choice + options: + - all + - googleapis + - otel + - sdk + dry_run: + description: "Dry run (build but do not upload)" + required: false + default: true + type: boolean + +concurrency: + group: ppa-publish-${{ github.ref }} + cancel-in-progress: false + +jobs: + upload-to-ppa: + name: "Build & Upload Source Packages" + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + + - name: Install packaging tools + run: | + sudo apt-get update + sudo apt-get install -y \ + devscripts \ + dput \ + debhelper \ + dpkg-dev \ + gnupg \ + cmake \ + protobuf-compiler \ + libprotobuf-dev + + - name: Import GPG key + if: ${{ !inputs.dry_run && github.event_name != 'workflow_dispatch' || (github.event_name == 'workflow_dispatch' && !inputs.dry_run) }} + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + run: | + echo "$GPG_PRIVATE_KEY" | gpg --batch --import + # Trust the key + GPG_FINGERPRINT=$(gpg --list-secret-keys --keyid-format long | grep -oP '(?<=sec\s{3}rsa4096/)[A-F0-9]+') + echo "${GPG_FINGERPRINT}:6:" | gpg --import-ownertrust + + - name: Determine parameters + id: params + run: | + # Series + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "series=${{ inputs.series }}" >> "$GITHUB_OUTPUT" + echo "packages=${{ inputs.packages }}" >> "$GITHUB_OUTPUT" + echo "dry_run=${{ inputs.dry_run }}" >> "$GITHUB_OUTPUT" + else + echo "series=noble" >> "$GITHUB_OUTPUT" + echo "packages=all" >> "$GITHUB_OUTPUT" + echo "dry_run=false" >> "$GITHUB_OUTPUT" + fi + + # GPG key ID + GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format long 2>/dev/null \ + | grep -oP '(?<=sec\s{3}rsa4096/)[A-F0-9]+' || echo "") + echo "gpg_key=$GPG_KEY_ID" >> "$GITHUB_OUTPUT" + + - name: Build and upload source packages + env: + LAUNCHPAD_PPA: ${{ secrets.LAUNCHPAD_PPA || 'ppa:ydb-team/ydb-cpp-sdk' }} + run: | + ARGS=() + ARGS+=(--ppa "$LAUNCHPAD_PPA") + ARGS+=(--series "${{ steps.params.outputs.series }}") + + case "${{ steps.params.outputs.packages }}" in + googleapis) ARGS+=(--googleapis) ;; + otel) ARGS+=(--otel) ;; + sdk) ARGS+=(--sdk) ;; + *) ARGS+=(--all) ;; + esac + + if [[ -n "${{ steps.params.outputs.gpg_key }}" ]]; then + ARGS+=(--gpg-key "${{ steps.params.outputs.gpg_key }}") + fi + + if [[ "${{ steps.params.outputs.dry_run }}" == "true" ]]; then + ARGS+=(--dry-run) + fi + + ./scripts/upload_to_ppa.sh "${ARGS[@]}" + + - name: Upload artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: source-packages + path: | + /tmp/tmp.*/*.dsc + /tmp/tmp.*/*.changes + /tmp/tmp.*/*.tar.gz + if-no-files-found: ignore + retention-days: 7 diff --git a/README.md b/README.md index 3ae363d2d0d..eb9aadb503a 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,57 @@ To smoke-test generated `.deb` packages with the sample consumer project: ./scripts/test_deb_packages.sh build-deb ``` +### Install from PPA + +Pre-built packages are available from the Launchpad PPA. This is the easiest +way to install the SDK on Ubuntu 24.04 (Noble): + +```bash +sudo add-apt-repository ppa:ydb-team/ydb-cpp-sdk +sudo apt-get update +sudo apt-get install libydb-cpp-dev +``` + +Optional packages: + +```bash +# IAM credentials plugin +sudo apt-get install libydb-cpp-iam-dev + +# OpenTelemetry plugins +sudo apt-get install libydb-cpp-otel-metrics-dev libydb-cpp-otel-tracing-dev +``` + +The PPA also provides `yandex-googleapis-api-common-protos` and +`yandex-opentelemetry-cpp-dev` which are installed automatically as +dependencies. + +After installation, use the SDK in your CMake project: + +```cmake +find_package(ydb-cpp-sdk REQUIRED COMPONENTS Driver Table Topic) +target_link_libraries(myapp PRIVATE YDB-CPP-SDK::Driver YDB-CPP-SDK::Table) +``` + +Pass `-DCMAKE_PREFIX_PATH=/usr/share/yandex` if the packages are installed +under the Yandex prefix. + +### Publish to PPA + +To upload new package versions to the PPA (maintainers only): + +```bash +# Dry run — build source packages without uploading +./scripts/upload_to_ppa.sh --all --dry-run --series noble + +# Actual upload (requires GPG key registered on Launchpad) +./scripts/upload_to_ppa.sh --all --gpg-key --series noble +``` + +Packages must be uploaded in order: `googleapis` → `otel` → `sdk`. +See [plans/ppa-setup.md](plans/ppa-setup.md) for detailed PPA creation +and GPG key setup instructions. + ### Test Specify a level of parallelism by passing the `-j` option into the command below (e.g. `-j$(nproc)`) diff --git a/plans/ppa-setup.md b/plans/ppa-setup.md new file mode 100644 index 00000000000..e497d82076a --- /dev/null +++ b/plans/ppa-setup.md @@ -0,0 +1,275 @@ +# PPA Setup Guide for YDB C++ SDK + +This document describes how to create and configure a Launchpad PPA for +distributing YDB C++ SDK Debian packages. + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [GPG Key Generation](#gpg-key-generation) +3. [Launchpad Account Setup](#launchpad-account-setup) +4. [PPA Creation](#ppa-creation) +5. [Local Configuration](#local-configuration) +6. [GitHub Secrets Configuration](#github-secrets-configuration) +7. [Testing the Upload Flow](#testing-the-upload-flow) +8. [Package Upload Order](#package-upload-order) +9. [Version Bumping](#version-bumping) +10. [Troubleshooting](#troubleshooting) + +--- + +## Prerequisites + +Install the required tools on your development machine (Ubuntu 22.04+): + +```bash +sudo apt-get update +sudo apt-get install -y \ + devscripts \ + dput \ + debhelper \ + dpkg-dev \ + gnupg \ + git +``` + +## GPG Key Generation + +Launchpad requires a GPG key to sign source packages. + +### Generate a new key + +```bash +gpg --full-generate-key +``` + +When prompted: +- **Key type**: RSA and RSA (default) +- **Key size**: 4096 bits +- **Expiration**: 0 (does not expire) — or set a reasonable expiration +- **Real name**: `YDB Team` (or your name) +- **Email**: `ydb-team@ydb.tech` (must match your Launchpad account email) +- **Passphrase**: choose a strong passphrase + +### Find your key ID + +```bash +gpg --list-secret-keys --keyid-format long +``` + +Output example: +``` +sec rsa4096/ABCDEF1234567890 2024-01-01 [SC] + FINGERPRINT1234567890ABCDEF1234567890ABCDEF +uid [ultimate] YDB Team +ssb rsa4096/1234567890ABCDEF 2024-01-01 [E] +``` + +Your **key ID** is `ABCDEF1234567890` (the part after `rsa4096/`). +Your **fingerprint** is the full 40-character hex string. + +### Upload key to Ubuntu keyserver + +```bash +gpg --keyserver keyserver.ubuntu.com --send-keys ABCDEF1234567890 +``` + +Wait a few minutes for propagation. + +### Export the private key (for CI) + +```bash +gpg --armor --export-secret-keys ABCDEF1234567890 > gpg-private-key.asc +``` + +**Keep this file secure!** It will be stored as a GitHub secret. + +## Launchpad Account Setup + +1. Go to https://launchpad.net and create an account (or sign in). +2. Navigate to https://launchpad.net/~/+editpgpkeys +3. Paste your GPG key **fingerprint** (40-character hex string). +4. Launchpad will send an encrypted email to verify ownership. +5. Decrypt the email and follow the confirmation link. + +## PPA Creation + +1. Go to https://launchpad.net/~/+activate-ppa +2. Fill in: + - **URL**: `ydb-cpp-sdk` + - **Display name**: `YDB C++ SDK` + - **Description**: + ``` + YDB C++ SDK development packages and dependencies. + + Packages: + - yandex-googleapis-api-common-protos: compiled Google API common proto specs + - yandex-opentelemetry-cpp-dev: OpenTelemetry C++ SDK + - libydb-cpp-dev: YDB C++ SDK core library + - libydb-cpp-iam-dev: YDB C++ SDK IAM plugin + - libydb-cpp-otel-metrics-dev: YDB C++ SDK OpenTelemetry metrics plugin + - libydb-cpp-otel-tracing-dev: YDB C++ SDK OpenTelemetry tracing plugin + ``` +3. Click **Activate**. + +The PPA URL will be: `ppa:YOUR_LAUNCHPAD_USERNAME/ydb-cpp-sdk` + +### Enable additional architectures (optional) + +By default, PPAs build for `amd64` and `i386`. To add `arm64`: + +1. Go to your PPA page on Launchpad +2. Click **Change details** +3. Under **Processors**, check `ARM ARMv8 (arm64)` + +## Local Configuration + +### Configure dput + +Create or edit `~/.dput.cf`: + +```ini +[ydb-cpp-sdk] +fqdn = ppa.launchpad.net +method = ftp +incoming = ~YOUR_LAUNCHPAD_USERNAME/ubuntu/ydb-cpp-sdk/ +login = anonymous +allow_unsigned_uploads = 0 +``` + +Alternatively, use the shorthand PPA syntax with `dput`: +```bash +dput ppa:YOUR_LAUNCHPAD_USERNAME/ydb-cpp-sdk +``` + +## GitHub Secrets Configuration + +For automated CI publishing, add these secrets to your GitHub repository +(Settings → Secrets and variables → Actions): + +| Secret Name | Description | How to obtain | +|---|---|---| +| `GPG_PRIVATE_KEY` | ASCII-armored GPG private key | `gpg --armor --export-secret-keys KEY_ID` | +| `GPG_PASSPHRASE` | Passphrase for the GPG key | The passphrase you chose during key generation | +| `LAUNCHPAD_PPA` | PPA identifier | `ppa:YOUR_USERNAME/ydb-cpp-sdk` | + +### Setting secrets via GitHub CLI + +```bash +gh secret set GPG_PRIVATE_KEY < gpg-private-key.asc +gh secret set GPG_PASSPHRASE --body "your-passphrase" +gh secret set LAUNCHPAD_PPA --body "ppa:ydb-team/ydb-cpp-sdk" +``` + +## Testing the Upload Flow + +### Dry run (no upload) + +```bash +# Build all source packages without uploading +./scripts/upload_to_ppa.sh --all --dry-run --series noble + +# Build only googleapis +./scripts/upload_to_ppa.sh --googleapis --dry-run + +# Build only SDK +./scripts/upload_to_ppa.sh --sdk --dry-run --series noble +``` + +### Actual upload + +```bash +# Upload all packages +./scripts/upload_to_ppa.sh --all --gpg-key ABCDEF1234567890 --series noble + +# Upload only googleapis first (required before SDK) +./scripts/upload_to_ppa.sh --googleapis --gpg-key ABCDEF1234567890 + +# Then upload opentelemetry-cpp +./scripts/upload_to_ppa.sh --otel --gpg-key ABCDEF1234567890 + +# Then upload SDK (after googleapis and otel are published) +./scripts/upload_to_ppa.sh --sdk --gpg-key ABCDEF1234567890 +``` + +### Via GitHub Actions + +1. Go to **Actions** → **Publish to PPA** +2. Click **Run workflow** +3. Select parameters: + - Series: `noble` + - Packages: `all` + - Dry run: `true` (for testing) +4. Review the output +5. Re-run with dry_run=`false` for actual upload + +## Package Upload Order + +**Important**: Packages must be uploaded and **published** in this order +because of build dependencies: + +1. `yandex-googleapis-api-common-protos` — no PPA dependencies +2. `yandex-opentelemetry-cpp` — no PPA dependencies +3. `ydb-cpp-sdk` — depends on both (1) and (2) at build time + +Wait for each package to be **published** on Launchpad before uploading +the next one. Launchpad build times vary but typically take 15–60 minutes. + +You can check build status at: +`https://launchpad.net/~YOUR_USERNAME/+archive/ubuntu/ydb-cpp-sdk/+packages` + +## Version Bumping + +### Rules for PPA versions + +- Each upload to the same series must have a **unique version**. +- Use the `~SERIES1` suffix convention: `3.18.0-1~noble1` +- For rebuilds, increment the suffix: `3.18.0-1~noble2`, `3.18.0-1~noble3` +- For multiple series, use different suffixes: `3.18.0-1~noble1`, `3.18.0-1~jammy1` + +### Updating the SDK version + +1. Edit `src/version.h` and update `YDB_SDK_VERSION` +2. The `generate-debian-directory.sh` script automatically extracts the version +3. Run `./scripts/upload_to_ppa.sh --sdk --series noble` + +### Updating the googleapis version + +1. Update the `CPACK_PACKAGE_VERSION` in `scripts/googleapis_deb/CMakeLists.txt` +2. Run `./scripts/upload_to_ppa.sh --googleapis --series noble` + +## Troubleshooting + +### "Signature verification failed" + +- Ensure your GPG key is uploaded to `keyserver.ubuntu.com` +- Ensure the key is registered on your Launchpad account +- Wait a few minutes after uploading the key for propagation + +### "Version already exists" + +- Increment the PPA suffix (e.g., `~noble1` → `~noble2`) +- Or delete the existing package from the PPA web interface + +### "Build dependency not satisfiable" + +- Ensure prerequisite packages are published in the PPA first +- Check that package names in `debian/control` match exactly + +### "Source format 3.0 (quilt) requires .orig.tar.gz" + +- Run without `--skip-orig` flag +- Or ensure the `.orig.tar.gz` was previously uploaded for this version + +### Build fails on Launchpad + +- Check the build log on the Launchpad PPA page +- Common issues: missing Build-Depends, network access during build + (Launchpad builds are sandboxed — no internet access) +- For opentelemetry-cpp: the FetchContent approach won't work on Launchpad; + the source must be included in the orig tarball + +### "Rejected: Could not find person" + +- The PPA identifier is wrong. Use `ppa:USERNAME/PPA_NAME` format. +- Verify with: `https://launchpad.net/~USERNAME/+archive/ubuntu/PPA_NAME` diff --git a/plans/ppa-upload-plan.md b/plans/ppa-upload-plan.md new file mode 100644 index 00000000000..6fa53ac66d2 --- /dev/null +++ b/plans/ppa-upload-plan.md @@ -0,0 +1,266 @@ +# Plan: PPA Upload Script & Launchpad PPA Setup + +## Overview + +This plan covers two deliverables: +1. A script (`scripts/upload_to_ppa.sh`) that builds Debian **source packages** and uploads them to a Launchpad PPA. +2. Complete instructions for creating and configuring the PPA on Launchpad. + +The PPA will host **two source packages**: +- `yandex-googleapis-api-common-protos` — compiled Google API common proto specs (static libs, `.pb.h`, `.pb.cc`, Python stubs) +- `ydb-cpp-sdk` — the SDK itself (4 binary packages: `libydb-cpp-dev`, `libydb-cpp-iam-dev`, `libydb-cpp-otel-metrics-dev`, `libydb-cpp-otel-tracing-dev`) + +## Architecture + +```mermaid +flowchart TD + A[Developer runs scripts/upload_to_ppa.sh] --> B{Which packages?} + B -->|--googleapis| C[Build googleapis source package] + B -->|--sdk| D[Build ydb-cpp-sdk source package] + B -->|--all or default| C + B -->|--all or default| D + + C --> C1[scripts/googleapis_deb/generate-debian-directory.sh] + C1 --> C2[debuild -S -sa] + C2 --> C3[dput ppa:ydb-team/ydb-cpp-sdk ..._source.changes] + + D --> D1[scripts/generate-debian-directory.sh] + D1 --> D2[debuild -S -sa] + D2 --> D3[dput ppa:ydb-team/ydb-cpp-sdk ..._source.changes] + + C3 --> E[Launchpad PPA builds .deb packages] + D3 --> E + + E --> F[Users: sudo add-apt-repository ppa:ydb-team/ydb-cpp-sdk] + F --> G[sudo apt install libydb-cpp-dev] +``` + +## Key Concepts: How Launchpad PPAs Work + +1. **PPAs only accept source packages** — Launchpad builds the `.deb` binaries on its own build farm. +2. Source packages consist of: `_.orig.tar.gz`, `_.debian.tar.xz`, `_.dsc`, `__source.changes`. +3. The `.changes` file must be **GPG-signed** with a key registered on Launchpad. +4. Upload is done via `dput` to `ppa:OWNER/PPA_NAME`. +5. Each source package needs a proper `debian/` directory with `changelog`, `control`, `rules`, `source/format`, and optionally `.install` files. + +## Detailed Steps + +### 1. Create `scripts/googleapis_deb/generate-debian-directory.sh` + +This script generates `debian/` metadata for the `yandex-googleapis-api-common-protos` source package. It must: + +- Extract version from `scripts/googleapis_deb/CMakeLists.txt` (currently `1.0.0`) +- Generate `debian/changelog` with proper format for the target Ubuntu series (e.g., `noble`) +- Generate `debian/control` with: + - `Source: yandex-googleapis-api-common-protos` + - `Build-Depends: debhelper-compat (= 13), cmake, protobuf-compiler, libprotobuf-dev` + - Binary package: `yandex-googleapis-api-common-protos` + - `Depends: libprotobuf-dev` +- Generate `debian/rules` using cmake buildsystem with `CMAKE_INSTALL_PREFIX=/usr/share/yandex` +- Generate `debian/source/format` as `3.0 (quilt)` +- Generate `.install` file mapping built artifacts + +**Key detail**: The CMakeLists.txt at `scripts/googleapis_deb/CMakeLists.txt` references `../../third_party/api-common-protos` — for the source package, the proto files must be included in the orig tarball. The script must handle this by creating a proper source tree. + +### 2. Update `scripts/generate-debian-directory.sh` + +The existing script already generates proper `debian/` metadata for the SDK. Minor updates needed: + +- Add `--series` parameter to target specific Ubuntu release (default: `noble` for 24.04) +- Ensure the changelog entry targets the correct series name instead of `unstable` +- Add `yandex-googleapis-api-common-protos` as a Build-Depends and runtime Depends for `libydb-cpp-dev` + +### 3. Create `scripts/upload_to_ppa.sh` + +Main upload script with the following interface: + +```bash +./scripts/upload_to_ppa.sh [OPTIONS] + +Options: + --ppa PPA_NAME PPA identifier (default: ppa:ydb-team/ydb-cpp-sdk) + --series SERIES Ubuntu series (default: noble) + --googleapis Build and upload only googleapis package + --sdk Build and upload only SDK packages + --all Build and upload all packages (default) + --gpg-key KEY_ID GPG key ID for signing + --dry-run Build source packages but do not upload + --skip-orig Skip creating .orig.tar.gz (use -sd instead of -sa) +``` + +The script will: + +1. **Validate prerequisites**: check for `debuild`, `dput`, `gpg`, `git` +2. **For googleapis package**: + - Create a temporary build directory + - Copy `scripts/googleapis_deb/` and `third_party/api-common-protos/` into it + - Run `scripts/googleapis_deb/generate-debian-directory.sh` + - Create `.orig.tar.gz` from the source tree + - Run `debuild -S -sa` (or `-sd` if `--skip-orig`) + - Run `dput $PPA ..._source.changes` +3. **For SDK package**: + - Run `scripts/generate-debian-directory.sh --series $SERIES` + - Create `.orig.tar.gz` from the git archive (excluding `.git`, build dirs) + - Run `debuild -S -sa` + - Run `dput $PPA ..._source.changes` + +### 4. Create `.github/workflows/ppa_publish.yaml` + +GitHub Actions workflow for automated PPA publishing on release tags: + +```yaml +on: + release: + types: [published] + workflow_dispatch: + inputs: + series: + description: Ubuntu series + default: noble + dry_run: + description: Dry run + type: boolean + default: true +``` + +The workflow will: +- Import GPG private key from GitHub secret `GPG_PRIVATE_KEY` +- Configure `dput` with PPA settings +- Run `scripts/upload_to_ppa.sh --all --series $SERIES --gpg-key $KEY_ID` +- Use `workflow_dispatch` for manual triggers with dry-run option + +### 5. Update `README.md` + +Add sections: +- **Using the PPA** — instructions for end users to add the PPA and install packages +- **Publishing to PPA** — instructions for maintainers on GPG setup and upload process + +### 6. Create `plans/ppa-setup.md` + +Comprehensive PPA setup documentation covering: +- Launchpad account creation +- GPG key generation and registration +- PPA creation on Launchpad +- GitHub secrets configuration +- Testing the upload flow + +## File Changes Summary + +| File | Action | Description | +|------|--------|-------------| +| `scripts/upload_to_ppa.sh` | **Create** | Main PPA upload script | +| `scripts/googleapis_deb/generate-debian-directory.sh` | **Create** | Debian metadata generator for googleapis package | +| `scripts/generate-debian-directory.sh` | **Modify** | Add --series param, fix changelog series | +| `.github/workflows/ppa_publish.yaml` | **Create** | CI workflow for automated PPA publishing | +| `README.md` | **Modify** | Add PPA usage and publishing instructions | +| `plans/ppa-setup.md` | **Create** | Detailed PPA setup documentation | + +## PPA Creation Instructions (for `plans/ppa-setup.md`) + +### Prerequisites + +1. **Launchpad account** at https://launchpad.net +2. **GPG key** (RSA 4096-bit recommended) +3. **Ubuntu development tools**: `sudo apt install devscripts dput debhelper gpg` + +### Step-by-step PPA Setup + +#### 1. Generate GPG Key + +```bash +gpg --full-generate-key +# Choose: RSA and RSA, 4096 bits, no expiration +# Name: YDB Team +# Email: +``` + +#### 2. Upload GPG Key to Ubuntu Keyserver + +```bash +gpg --keyserver keyserver.ubuntu.com --send-keys +``` + +#### 3. Register GPG Key on Launchpad + +- Go to https://launchpad.net/~/+editpgpkeys +- Paste the GPG key fingerprint +- Confirm via email + +#### 4. Create PPA on Launchpad + +- Go to https://launchpad.net/~/+activate-ppa +- Name: `ydb-cpp-sdk` +- Display name: `YDB C++ SDK` +- Description: `YDB C++ SDK development packages and dependencies` +- Distribution: `Ubuntu` +- Architectures: `amd64`, `arm64` (optional) + +#### 5. Configure `~/.dput.cf` + +```ini +[ydb-cpp-sdk] +fqdn = ppa.launchpad.net +method = ftp +incoming = ~ydb-team/ubuntu/ydb-cpp-sdk/ +login = anonymous +allow_unsigned_uploads = 0 +``` + +Or use the shorthand: `dput ppa:ydb-team/ydb-cpp-sdk ` + +#### 6. Configure GitHub Secrets + +For CI automation, add these secrets to the GitHub repository: + +| Secret | Description | +|--------|-------------| +| `GPG_PRIVATE_KEY` | ASCII-armored GPG private key (`gpg --armor --export-secret-keys KEY_ID`) | +| `GPG_PASSPHRASE` | Passphrase for the GPG key | +| `LAUNCHPAD_PPA` | PPA identifier (e.g., `ppa:ydb-team/ydb-cpp-sdk`) | + +#### 7. Test Upload + +```bash +# Dry run (builds source packages without uploading) +./scripts/upload_to_ppa.sh --all --dry-run + +# Actual upload +./scripts/upload_to_ppa.sh --all --gpg-key +``` + +#### 8. End-User Installation + +Once packages are published: + +```bash +sudo add-apt-repository ppa:ydb-team/ydb-cpp-sdk +sudo apt update +sudo apt install libydb-cpp-dev libydb-cpp-iam-dev +``` + +## Package Dependency Chain + +```mermaid +flowchart LR + A[yandex-googleapis-api-common-protos] --> B[libydb-cpp-dev] + B --> C[libydb-cpp-iam-dev] + B --> D[libydb-cpp-otel-metrics-dev] + B --> E[libydb-cpp-otel-tracing-dev] + + F[libprotobuf-dev] --> A + F --> B + G[libgrpc++-dev] --> B + H[opentelemetry-cpp] --> D + H --> E +``` + +## Important Notes + +1. **PPA builds happen on Launchpad servers** — all Build-Depends must be available in the target Ubuntu release or in the same PPA. +2. **googleapis package must be uploaded first** — the SDK depends on it at build time. +3. **OpenTelemetry**: Ubuntu 24.04 does not ship `opentelemetry-cpp` packages. Options: + - Build and upload an `opentelemetry-cpp` package to the same PPA + - Disable OTel plugins in the PPA build (simpler, but loses functionality) + - Use vendored/FetchContent approach in the PPA build +4. **Version bumping**: Each upload to the same series must have a unique version. Use `~ppa1`, `~ppa2` suffixes for rebuilds. +5. **The `CPACK_PACKAGING_INSTALL_PREFIX`** in `scripts/googleapis_deb/CMakeLists.txt` is set to `/usr/share/yandex` — this must match the `CMAKE_INSTALL_PREFIX` and `CMAKE_PREFIX_PATH` used by the SDK build. diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index ec3ee29de66..0a2f2c9a79d 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -4,6 +4,28 @@ set -e SCRIPT_DIR=$(dirname "$(realpath "$0")") SOURCE_DIR=$(realpath "$SCRIPT_DIR/..") +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +SERIES="noble" +while [[ $# -gt 0 ]]; do + case "$1" in + --series) + SERIES="$2" + shift 2 + ;; + --series=*) + SERIES="${1#*=}" + shift + ;; + *) + echo "Unknown option: $1" >&2 + echo "Usage: $0 [--series ]" >&2 + exit 1 + ;; + esac +done + cd "$SOURCE_DIR" VERSION=$(grep -E 'YDB_SDK_VERSION = "[0-9]+\.[0-9]+\.[0-9]+"' src/version.h | sed -E 's/.*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/') @@ -18,22 +40,22 @@ mkdir -p debian/ CHANGELOG_FILE="debian/changelog" -DEB_VERSION="${VERSION}-1" +DEB_VERSION="${VERSION}-1~${SERIES}1" cat < "$CHANGELOG_FILE" -ydb-cpp-sdk (${DEB_VERSION}) unstable; urgency=low +ydb-cpp-sdk (${DEB_VERSION}) ${SERIES}; urgency=low * Automated package build * Git commit: ${GIT_COMMIT} - -- YDB Team $(date -R) + -- YDB Team $(date -R) EOF_CHANGELOG cat < debian/control Source: ydb-cpp-sdk Section: libdevel Priority: optional -Maintainer: YDB Team +Maintainer: YDB Team Build-Depends: debhelper-compat (= 13), cmake, git, @@ -60,7 +82,8 @@ Build-Depends: debhelper-compat (= 13), libre2-dev, libc-ares-dev, rapidjson-dev, - yandex-googleapis-api-common-protos + yandex-googleapis-api-common-protos, + yandex-opentelemetry-cpp-dev Standards-Version: 4.6.2 Homepage: https://ydb.tech Rules-Requires-Root: no @@ -96,13 +119,13 @@ Description: YDB C++ SDK IAM plugin development files Package: libydb-cpp-otel-metrics-dev Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}), yandex-opentelemetry-cpp-dev Description: YDB C++ SDK OpenTelemetry metrics plugin development files Static library and headers for YDB C++ SDK OpenTelemetry metrics plugin. Package: libydb-cpp-otel-tracing-dev Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}), yandex-opentelemetry-cpp-dev Description: YDB C++ SDK OpenTelemetry tracing plugin development files Static library and headers for YDB C++ SDK OpenTelemetry tracing plugin. EOF_CONTROL @@ -141,10 +164,6 @@ debian/tmp/usr/share/yandex/lib/*/cmake/base64 usr/share/yandex/lib/*/cmake/ debian/tmp/usr/share/yandex/include/picojson usr/share/yandex/include/ debian/tmp/usr/share/yandex/include/jwt-cpp usr/share/yandex/include/ debian/tmp/usr/share/yandex/cmake/jwt-cpp* usr/share/yandex/cmake/ -debian/tmp/usr/share/yandex/include/opentelemetry usr/share/yandex/include/ -debian/tmp/usr/share/yandex/lib/*/libopentelemetry_* usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/lib/*/cmake/opentelemetry-cpp usr/share/yandex/lib/*/cmake/ -debian/tmp/usr/share/yandex/lib/*/pkgconfig/opentelemetry_*.pc usr/share/yandex/lib/*/pkgconfig/ EOF_INSTALL cat < debian/libydb-cpp-iam-dev.install @@ -165,4 +184,4 @@ EOF_INSTALL mkdir -p debian/source echo "3.0 (quilt)" > debian/source/format -echo "Generated debian directory for version ${DEB_VERSION}" +echo "Generated debian directory for version ${DEB_VERSION} (series: ${SERIES})" diff --git a/scripts/googleapis_deb/generate-debian-directory.sh b/scripts/googleapis_deb/generate-debian-directory.sh new file mode 100755 index 00000000000..5b6a10f22ff --- /dev/null +++ b/scripts/googleapis_deb/generate-debian-directory.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Generate debian/ directory for yandex-googleapis-api-common-protos source package. +# This script is intended to be run from the repository root or from +# scripts/googleapis_deb/ itself. It produces a debian/ directory inside +# scripts/googleapis_deb/ that is suitable for dpkg-buildpackage / debuild. +set -euo pipefail + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +REPO_ROOT=$(realpath "$SCRIPT_DIR/../..") + +SERIES="${1:-noble}" + +# --------------------------------------------------------------------------- +# Version +# --------------------------------------------------------------------------- +VERSION=$(grep -oP 'CPACK_PACKAGE_VERSION\s+"\K[^"]+' "$SCRIPT_DIR/CMakeLists.txt" || echo "1.0.0") +GIT_COMMIT=$(cd "$REPO_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") +DEB_VERSION="${VERSION}-1~${SERIES}1" + +# --------------------------------------------------------------------------- +# debian/ skeleton +# --------------------------------------------------------------------------- +DEB_DIR="$SCRIPT_DIR/debian" +mkdir -p "$DEB_DIR/source" + +# -- changelog -------------------------------------------------------------- +cat > "$DEB_DIR/changelog" < $(date -R) +EOF + +# -- control ---------------------------------------------------------------- +cat > "$DEB_DIR/control" <<'EOF' +Source: yandex-googleapis-api-common-protos +Section: libdevel +Priority: optional +Maintainer: YDB Team +Build-Depends: debhelper-compat (= 13), + cmake, + protobuf-compiler, + libprotobuf-dev +Standards-Version: 4.6.2 +Homepage: https://github.com/googleapis/api-common-protos +Rules-Requires-Root: no + +Package: yandex-googleapis-api-common-protos +Architecture: any +Depends: ${misc:Depends}, libprotobuf-dev +Description: Google API common proto compiled libraries (Yandex build) + Pre-compiled C++ static libraries and headers generated from the + googleapis/api-common-protos proto definitions. Installed under + /usr/share/yandex so that YDB C++ SDK and other Yandex packages can + find_package(yandex-googleapis-api-common-protos). +EOF + +# -- rules ------------------------------------------------------------------ +cat > "$DEB_DIR/rules" <<'RULES' +#!/usr/bin/make -f + +export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +%: + dh $@ --buildsystem=cmake + +override_dh_auto_configure: + dh_auto_configure -- \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex + +override_dh_auto_install: + dh_auto_install --destdir=debian/tmp +RULES +chmod +x "$DEB_DIR/rules" + +# -- install ---------------------------------------------------------------- +cat > "$DEB_DIR/yandex-googleapis-api-common-protos.install" <<'EOF' +debian/tmp/usr/share/yandex/lib/*/libapi-common-protos.a usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/include/google usr/share/yandex/include/ +debian/tmp/usr/share/yandex/lib/*/cmake/yandex-googleapis-api-common-protos usr/share/yandex/lib/cmake/ +EOF + +# -- source/format ---------------------------------------------------------- +echo "3.0 (quilt)" > "$DEB_DIR/source/format" + +echo "Generated debian directory for yandex-googleapis-api-common-protos ${DEB_VERSION} (series: ${SERIES})" diff --git a/scripts/otel_deb/CMakeLists.txt b/scripts/otel_deb/CMakeLists.txt new file mode 100644 index 00000000000..da848d2b3b7 --- /dev/null +++ b/scripts/otel_deb/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.14) +project(yandex-opentelemetry-cpp VERSION 1.12.0) + +set(CMAKE_CXX_STANDARD 20) + +include(FetchContent) + +set(BUILD_TESTING OFF CACHE BOOL "" FORCE) +set(WITH_OTLP_HTTP OFF CACHE BOOL "" FORCE) +set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE) +set(WITH_PROMETHEUS OFF CACHE BOOL "" FORCE) +set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE) +set(OTELCPP_MAINTAINER_MODE OFF CACHE BOOL "" FORCE) +set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "" FORCE) +set(WITH_STL "CXX20" CACHE STRING "" FORCE) +set(OPENTELEMETRY_INSTALL ON CACHE BOOL "" FORCE) + +FetchContent_Declare( + opentelemetry-cpp + GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git + GIT_TAG v1.12.0 +) +FetchContent_MakeAvailable(opentelemetry-cpp) diff --git a/scripts/otel_deb/generate-debian-directory.sh b/scripts/otel_deb/generate-debian-directory.sh new file mode 100755 index 00000000000..cb96d28b0fd --- /dev/null +++ b/scripts/otel_deb/generate-debian-directory.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Generate debian/ directory for yandex-opentelemetry-cpp source package. +# This wraps the upstream opentelemetry-cpp v1.12.0 into a Debian source +# package that installs under /usr/share/yandex so it can be consumed by +# the YDB C++ SDK packaging build. +set -euo pipefail + +SCRIPT_DIR=$(dirname "$(realpath "$0")") + +SERIES="${1:-noble}" +OTEL_VERSION="1.12.0" +GIT_COMMIT="v${OTEL_VERSION}" +DEB_VERSION="${OTEL_VERSION}-1~${SERIES}1" + +# --------------------------------------------------------------------------- +# debian/ skeleton +# --------------------------------------------------------------------------- +DEB_DIR="$SCRIPT_DIR/debian" +mkdir -p "$DEB_DIR/source" + +# -- changelog -------------------------------------------------------------- +cat > "$DEB_DIR/changelog" < $(date -R) +EOF + +# -- control ---------------------------------------------------------------- +cat > "$DEB_DIR/control" <<'EOF' +Source: yandex-opentelemetry-cpp +Section: libdevel +Priority: optional +Maintainer: YDB Team +Build-Depends: debhelper-compat (= 13), + cmake, + git, + pkg-config, + libcurl4-openssl-dev, + nlohmann-json3-dev +Standards-Version: 4.6.2 +Homepage: https://opentelemetry.io/docs/languages/cpp/ +Rules-Requires-Root: no + +Package: yandex-opentelemetry-cpp-dev +Architecture: any +Depends: ${misc:Depends} +Description: OpenTelemetry C++ SDK (Yandex build) + Static libraries, headers and CMake package files for the OpenTelemetry + C++ SDK. Installed under /usr/share/yandex so that YDB C++ SDK and + other Yandex packages can find_package(opentelemetry-cpp). +EOF + +# -- rules ------------------------------------------------------------------ +cat > "$DEB_DIR/rules" <<'RULES' +#!/usr/bin/make -f + +export DEB_BUILD_MAINT_OPTIONS = hardening=+all + +%: + dh $@ --buildsystem=cmake + +override_dh_auto_configure: + dh_auto_configure -- \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + +override_dh_auto_install: + dh_auto_install --destdir=debian/tmp +RULES +chmod +x "$DEB_DIR/rules" + +# -- install ---------------------------------------------------------------- +cat > "$DEB_DIR/yandex-opentelemetry-cpp-dev.install" <<'EOF' +debian/tmp/usr/share/yandex/lib/*/libopentelemetry_*.a usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/include/opentelemetry usr/share/yandex/include/ +debian/tmp/usr/share/yandex/lib/*/cmake/opentelemetry-cpp usr/share/yandex/lib/cmake/ +debian/tmp/usr/share/yandex/lib/*/pkgconfig/opentelemetry_*.pc usr/share/yandex/lib/pkgconfig/ +EOF + +# -- source/format ---------------------------------------------------------- +echo "3.0 (quilt)" > "$DEB_DIR/source/format" + +echo "Generated debian directory for yandex-opentelemetry-cpp ${DEB_VERSION} (series: ${SERIES})" diff --git a/scripts/upload_to_ppa.sh b/scripts/upload_to_ppa.sh new file mode 100755 index 00000000000..be4a3ac2722 --- /dev/null +++ b/scripts/upload_to_ppa.sh @@ -0,0 +1,300 @@ +#!/bin/bash +# ============================================================================= +# upload_to_ppa.sh — Build Debian source packages and upload them to a +# Launchpad PPA. +# +# Usage: +# ./scripts/upload_to_ppa.sh [OPTIONS] +# +# Options: +# --ppa PPA PPA identifier (default: ppa:ydb-team/ydb-cpp-sdk) +# --series SERIES Ubuntu series name (default: noble) +# --googleapis Build/upload only the googleapis package +# --otel Build/upload only the opentelemetry-cpp package +# --sdk Build/upload only the ydb-cpp-sdk packages +# --all Build/upload all packages (default) +# --gpg-key KEY_ID GPG key fingerprint for signing +# --dry-run Build source packages but do not upload +# --skip-orig Pass -sd to debuild (reuse existing .orig.tar.gz) +# --help Show this help message +# ============================================================================= +set -euo pipefail + +SCRIPT_DIR=$(dirname "$(realpath "$0")") +REPO_ROOT=$(realpath "$SCRIPT_DIR/..") + +# ---- defaults --------------------------------------------------------------- +PPA="ppa:ydb-team/ydb-cpp-sdk" +SERIES="noble" +GPG_KEY="" +DRY_RUN=false +SKIP_ORIG=false +BUILD_GOOGLEAPIS=false +BUILD_OTEL=false +BUILD_SDK=false +BUILD_ALL=true + +# ---- parse arguments -------------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --ppa) PPA="$2"; shift 2 ;; + --ppa=*) PPA="${1#*=}"; shift ;; + --series) SERIES="$2"; shift 2 ;; + --series=*) SERIES="${1#*=}"; shift ;; + --gpg-key) GPG_KEY="$2"; shift 2 ;; + --gpg-key=*) GPG_KEY="${1#*=}"; shift ;; + --googleapis) BUILD_GOOGLEAPIS=true; BUILD_ALL=false; shift ;; + --otel) BUILD_OTEL=true; BUILD_ALL=false; shift ;; + --sdk) BUILD_SDK=true; BUILD_ALL=false; shift ;; + --all) BUILD_ALL=true; shift ;; + --dry-run) DRY_RUN=true; shift ;; + --skip-orig) SKIP_ORIG=true; shift ;; + --help) + head -n 20 "$0" | grep '^#' | sed 's/^# \?//' + exit 0 + ;; + *) + echo "Error: unknown option '$1'" >&2 + echo "Run $0 --help for usage." >&2 + exit 1 + ;; + esac +done + +if $BUILD_ALL; then + BUILD_GOOGLEAPIS=true + BUILD_OTEL=true + BUILD_SDK=true +fi + +# ---- prerequisites ---------------------------------------------------------- +for cmd in debuild dput gpg git tar; do + if ! command -v "$cmd" &>/dev/null; then + echo "Error: '$cmd' is not installed. Install it first:" >&2 + echo " sudo apt install devscripts dput gnupg git" >&2 + exit 1 + fi +done + +DEBUILD_SIGN_ARGS=() +if [[ -n "$GPG_KEY" ]]; then + DEBUILD_SIGN_ARGS=(-k"$GPG_KEY") +fi + +DEBUILD_ORIG_FLAG="-sa" +if $SKIP_ORIG; then + DEBUILD_ORIG_FLAG="-sd" +fi + +# ---- helper: upload or dry-run ---------------------------------------------- +do_upload() { + local changes_file="$1" + if $DRY_RUN; then + echo "[dry-run] Would upload: $changes_file" + echo "[dry-run] dput $PPA $changes_file" + else + echo "Uploading $changes_file to $PPA ..." + dput "$PPA" "$changes_file" + fi +} + +# ============================================================================= +# 1. yandex-googleapis-api-common-protos +# ============================================================================= +if $BUILD_GOOGLEAPIS; then + echo "============================================================" + echo "Building source package: yandex-googleapis-api-common-protos" + echo "============================================================" + + GAPI_SRC="$SCRIPT_DIR/googleapis_deb" + GAPI_VERSION=$(grep -oP 'CPACK_PACKAGE_VERSION\s+"\K[^"]+' "$GAPI_SRC/CMakeLists.txt" || echo "1.0.0") + GAPI_DEB_VERSION="${GAPI_VERSION}-1~${SERIES}1" + GAPI_PKG="yandex-googleapis-api-common-protos" + + WORK_DIR=$(mktemp -d) + trap "rm -rf $WORK_DIR" EXIT + + GAPI_BUILD_DIR="$WORK_DIR/${GAPI_PKG}-${GAPI_VERSION}" + mkdir -p "$GAPI_BUILD_DIR" + + # Copy CMakeLists.txt and config template + cp "$GAPI_SRC/CMakeLists.txt" "$GAPI_BUILD_DIR/" + cp "$GAPI_SRC/yandex-googleapis-api-common-protosConfig.cmake.in" "$GAPI_BUILD_DIR/" + + # Copy proto sources (the submodule content) + if [[ -d "$REPO_ROOT/third_party/api-common-protos/google" ]]; then + mkdir -p "$GAPI_BUILD_DIR/third_party/api-common-protos" + cp -r "$REPO_ROOT/third_party/api-common-protos/google" \ + "$GAPI_BUILD_DIR/third_party/api-common-protos/" + else + echo "Error: third_party/api-common-protos/google not found." >&2 + echo "Initialize git submodules: git submodule update --init" >&2 + exit 1 + fi + + # Fix CMakeLists.txt PROTOS_DIR to point to local copy + sed -i 's|"\${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/api-common-protos"|"${CMAKE_CURRENT_SOURCE_DIR}/third_party/api-common-protos"|' \ + "$GAPI_BUILD_DIR/CMakeLists.txt" + + # Generate debian/ directory + bash "$GAPI_SRC/generate-debian-directory.sh" "$SERIES" + cp -r "$GAPI_SRC/debian" "$GAPI_BUILD_DIR/" + + # Create .orig.tar.gz + ORIG_TAR="$WORK_DIR/${GAPI_PKG}_${GAPI_VERSION}.orig.tar.gz" + tar -czf "$ORIG_TAR" -C "$WORK_DIR" "${GAPI_PKG}-${GAPI_VERSION}" + + # Build source package + cd "$GAPI_BUILD_DIR" + debuild -S "$DEBUILD_ORIG_FLAG" "${DEBUILD_SIGN_ARGS[@]}" + + # Upload + CHANGES_FILE="$WORK_DIR/${GAPI_PKG}_${GAPI_DEB_VERSION}_source.changes" + if [[ -f "$CHANGES_FILE" ]]; then + do_upload "$CHANGES_FILE" + else + echo "Warning: expected changes file not found: $CHANGES_FILE" >&2 + # Try to find it + CHANGES_FILE=$(ls "$WORK_DIR"/${GAPI_PKG}_*_source.changes 2>/dev/null | head -1) + if [[ -n "$CHANGES_FILE" ]]; then + do_upload "$CHANGES_FILE" + else + echo "Error: no _source.changes file found for googleapis." >&2 + exit 1 + fi + fi + + cd "$REPO_ROOT" + echo "googleapis source package built successfully." + echo "" +fi + +# ============================================================================= +# 2. yandex-opentelemetry-cpp +# ============================================================================= +if $BUILD_OTEL; then + echo "============================================================" + echo "Building source package: yandex-opentelemetry-cpp" + echo "============================================================" + + OTEL_SRC="$SCRIPT_DIR/otel_deb" + OTEL_VERSION="1.12.0" + OTEL_DEB_VERSION="${OTEL_VERSION}-1~${SERIES}1" + OTEL_PKG="yandex-opentelemetry-cpp" + + WORK_DIR_OTEL=$(mktemp -d) + trap "rm -rf $WORK_DIR_OTEL" EXIT + + OTEL_BUILD_DIR="$WORK_DIR_OTEL/${OTEL_PKG}-${OTEL_VERSION}" + mkdir -p "$OTEL_BUILD_DIR" + + # Copy CMakeLists.txt + cp "$OTEL_SRC/CMakeLists.txt" "$OTEL_BUILD_DIR/" + + # Generate debian/ directory + bash "$OTEL_SRC/generate-debian-directory.sh" "$SERIES" + cp -r "$OTEL_SRC/debian" "$OTEL_BUILD_DIR/" + + # Create .orig.tar.gz + ORIG_TAR_OTEL="$WORK_DIR_OTEL/${OTEL_PKG}_${OTEL_VERSION}.orig.tar.gz" + tar -czf "$ORIG_TAR_OTEL" -C "$WORK_DIR_OTEL" "${OTEL_PKG}-${OTEL_VERSION}" + + # Build source package + cd "$OTEL_BUILD_DIR" + debuild -S "$DEBUILD_ORIG_FLAG" "${DEBUILD_SIGN_ARGS[@]}" + + # Upload + CHANGES_FILE_OTEL="$WORK_DIR_OTEL/${OTEL_PKG}_${OTEL_DEB_VERSION}_source.changes" + if [[ -f "$CHANGES_FILE_OTEL" ]]; then + do_upload "$CHANGES_FILE_OTEL" + else + CHANGES_FILE_OTEL=$(ls "$WORK_DIR_OTEL"/${OTEL_PKG}_*_source.changes 2>/dev/null | head -1) + if [[ -n "$CHANGES_FILE_OTEL" ]]; then + do_upload "$CHANGES_FILE_OTEL" + else + echo "Error: no _source.changes file found for opentelemetry-cpp." >&2 + exit 1 + fi + fi + + cd "$REPO_ROOT" + echo "opentelemetry-cpp source package built successfully." + echo "" +fi + +# ============================================================================= +# 3. ydb-cpp-sdk +# ============================================================================= +if $BUILD_SDK; then + echo "============================================================" + echo "Building source package: ydb-cpp-sdk" + echo "============================================================" + + SDK_VERSION=$(grep -E 'YDB_SDK_VERSION = "[0-9]+\.[0-9]+\.[0-9]+"' "$REPO_ROOT/src/version.h" \ + | sed -E 's/.*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/') + if [[ -z "$SDK_VERSION" ]]; then + echo "Error: Could not extract YDB_SDK_VERSION from src/version.h" >&2 + exit 1 + fi + SDK_DEB_VERSION="${SDK_VERSION}-1~${SERIES}1" + SDK_PKG="ydb-cpp-sdk" + + WORK_DIR_SDK=$(mktemp -d) + trap "rm -rf $WORK_DIR_SDK" EXIT + + SDK_BUILD_DIR="$WORK_DIR_SDK/${SDK_PKG}-${SDK_VERSION}" + + # Create source tarball from git archive (includes submodules content) + echo "Creating source tarball..." + cd "$REPO_ROOT" + git archive --format=tar --prefix="${SDK_PKG}-${SDK_VERSION}/" HEAD \ + | tar -xf - -C "$WORK_DIR_SDK" + + # Copy submodule contents (git archive doesn't include them) + if [[ -d "$REPO_ROOT/third_party/api-common-protos/google" ]]; then + mkdir -p "$SDK_BUILD_DIR/third_party/api-common-protos" + cp -r "$REPO_ROOT/third_party/api-common-protos/google" \ + "$SDK_BUILD_DIR/third_party/api-common-protos/" + fi + if [[ -d "$REPO_ROOT/third_party/FastLZ" ]]; then + mkdir -p "$SDK_BUILD_DIR/third_party/FastLZ" + cp -r "$REPO_ROOT/third_party/FastLZ/"* "$SDK_BUILD_DIR/third_party/FastLZ/" 2>/dev/null || true + fi + + # Generate debian/ directory + bash "$SCRIPT_DIR/generate-debian-directory.sh" --series "$SERIES" + cp -r "$REPO_ROOT/debian" "$SDK_BUILD_DIR/" + + # Create .orig.tar.gz + ORIG_TAR_SDK="$WORK_DIR_SDK/${SDK_PKG}_${SDK_VERSION}.orig.tar.gz" + tar -czf "$ORIG_TAR_SDK" -C "$WORK_DIR_SDK" "${SDK_PKG}-${SDK_VERSION}" + + # Build source package + cd "$SDK_BUILD_DIR" + debuild -S "$DEBUILD_ORIG_FLAG" "${DEBUILD_SIGN_ARGS[@]}" + + # Upload + CHANGES_FILE_SDK="$WORK_DIR_SDK/${SDK_PKG}_${SDK_DEB_VERSION}_source.changes" + if [[ -f "$CHANGES_FILE_SDK" ]]; then + do_upload "$CHANGES_FILE_SDK" + else + CHANGES_FILE_SDK=$(ls "$WORK_DIR_SDK"/${SDK_PKG}_*_source.changes 2>/dev/null | head -1) + if [[ -n "$CHANGES_FILE_SDK" ]]; then + do_upload "$CHANGES_FILE_SDK" + else + echo "Error: no _source.changes file found for ydb-cpp-sdk." >&2 + exit 1 + fi + fi + + cd "$REPO_ROOT" + echo "ydb-cpp-sdk source package built successfully." + echo "" +fi + +echo "============================================================" +echo "Done." +if $DRY_RUN; then + echo "(dry-run mode — nothing was uploaded)" +fi +echo "============================================================" From f1b5cb4d3e7920fe05e237d816535150283f8379 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Thu, 21 May 2026 13:06:07 +0300 Subject: [PATCH 12/13] finish .debs with a releaseworkflow --- .github/workflows/ppa_publish.yaml | 121 ------- .github/workflows/release_publish.yaml | 128 ++++++++ README.md | 56 ++-- plans/ppa-setup.md | 275 ---------------- plans/ppa-upload-plan.md | 266 ---------------- .../generate-debian-directory.sh | 89 ------ scripts/otel_deb/CMakeLists.txt | 23 -- scripts/otel_deb/generate-debian-directory.sh | 87 ----- scripts/upload_to_ppa.sh | 300 ------------------ 9 files changed, 147 insertions(+), 1198 deletions(-) delete mode 100644 .github/workflows/ppa_publish.yaml create mode 100644 .github/workflows/release_publish.yaml delete mode 100644 plans/ppa-setup.md delete mode 100644 plans/ppa-upload-plan.md delete mode 100755 scripts/googleapis_deb/generate-debian-directory.sh delete mode 100644 scripts/otel_deb/CMakeLists.txt delete mode 100755 scripts/otel_deb/generate-debian-directory.sh delete mode 100755 scripts/upload_to_ppa.sh diff --git a/.github/workflows/ppa_publish.yaml b/.github/workflows/ppa_publish.yaml deleted file mode 100644 index 53fd813c280..00000000000 --- a/.github/workflows/ppa_publish.yaml +++ /dev/null @@ -1,121 +0,0 @@ -name: Publish to PPA - -on: - release: - types: [published] - workflow_dispatch: - inputs: - series: - description: "Ubuntu series (e.g. noble, jammy)" - required: false - default: "noble" - packages: - description: "Which packages to upload" - required: false - default: "all" - type: choice - options: - - all - - googleapis - - otel - - sdk - dry_run: - description: "Dry run (build but do not upload)" - required: false - default: true - type: boolean - -concurrency: - group: ppa-publish-${{ github.ref }} - cancel-in-progress: false - -jobs: - upload-to-ppa: - name: "Build & Upload Source Packages" - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - - name: Install packaging tools - run: | - sudo apt-get update - sudo apt-get install -y \ - devscripts \ - dput \ - debhelper \ - dpkg-dev \ - gnupg \ - cmake \ - protobuf-compiler \ - libprotobuf-dev - - - name: Import GPG key - if: ${{ !inputs.dry_run && github.event_name != 'workflow_dispatch' || (github.event_name == 'workflow_dispatch' && !inputs.dry_run) }} - env: - GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} - GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - run: | - echo "$GPG_PRIVATE_KEY" | gpg --batch --import - # Trust the key - GPG_FINGERPRINT=$(gpg --list-secret-keys --keyid-format long | grep -oP '(?<=sec\s{3}rsa4096/)[A-F0-9]+') - echo "${GPG_FINGERPRINT}:6:" | gpg --import-ownertrust - - - name: Determine parameters - id: params - run: | - # Series - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - echo "series=${{ inputs.series }}" >> "$GITHUB_OUTPUT" - echo "packages=${{ inputs.packages }}" >> "$GITHUB_OUTPUT" - echo "dry_run=${{ inputs.dry_run }}" >> "$GITHUB_OUTPUT" - else - echo "series=noble" >> "$GITHUB_OUTPUT" - echo "packages=all" >> "$GITHUB_OUTPUT" - echo "dry_run=false" >> "$GITHUB_OUTPUT" - fi - - # GPG key ID - GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format long 2>/dev/null \ - | grep -oP '(?<=sec\s{3}rsa4096/)[A-F0-9]+' || echo "") - echo "gpg_key=$GPG_KEY_ID" >> "$GITHUB_OUTPUT" - - - name: Build and upload source packages - env: - LAUNCHPAD_PPA: ${{ secrets.LAUNCHPAD_PPA || 'ppa:ydb-team/ydb-cpp-sdk' }} - run: | - ARGS=() - ARGS+=(--ppa "$LAUNCHPAD_PPA") - ARGS+=(--series "${{ steps.params.outputs.series }}") - - case "${{ steps.params.outputs.packages }}" in - googleapis) ARGS+=(--googleapis) ;; - otel) ARGS+=(--otel) ;; - sdk) ARGS+=(--sdk) ;; - *) ARGS+=(--all) ;; - esac - - if [[ -n "${{ steps.params.outputs.gpg_key }}" ]]; then - ARGS+=(--gpg-key "${{ steps.params.outputs.gpg_key }}") - fi - - if [[ "${{ steps.params.outputs.dry_run }}" == "true" ]]; then - ARGS+=(--dry-run) - fi - - ./scripts/upload_to_ppa.sh "${ARGS[@]}" - - - name: Upload artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: source-packages - path: | - /tmp/tmp.*/*.dsc - /tmp/tmp.*/*.changes - /tmp/tmp.*/*.tar.gz - if-no-files-found: ignore - retention-days: 7 diff --git a/.github/workflows/release_publish.yaml b/.github/workflows/release_publish.yaml new file mode 100644 index 00000000000..6ab6dd96c17 --- /dev/null +++ b/.github/workflows/release_publish.yaml @@ -0,0 +1,128 @@ +name: Publish release artifacts + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: "Release tag to attach assets to (e.g. v1.2.3)" + required: true + type: string + +concurrency: + group: release-publish-${{ github.event.release.tag_name || inputs.tag }} + cancel-in-progress: false + +permissions: + contents: write + +jobs: + build-and-upload-debs: + name: "Build .deb packages and attach to release (Ubuntu 24.04 / noble)" + runs-on: ubuntu-latest + steps: + - name: Resolve release tag + id: tag + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT" + echo "ref=${{ inputs.tag }}" >> "$GITHUB_OUTPUT" + else + echo "tag=${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT" + echo "ref=${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + ref: ${{ steps.tag.outputs.ref }} + + - name: Build .deb packages in Ubuntu 24.04 container + shell: bash + run: | + mkdir -p artifacts + docker run --rm --network host \ + -v "$PWD:/source" \ + ubuntu:24.04 \ + bash -c ' + set -e + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + git \ + libidn11-dev \ + libssl-dev \ + zlib1g-dev \ + libprotobuf-dev \ + protobuf-compiler \ + libgrpc++-dev \ + protobuf-compiler-grpc \ + libbrotli-dev \ + liblz4-dev \ + libzstd-dev \ + libbz2-dev \ + libxxhash-dev \ + libsnappy-dev \ + libdouble-conversion-dev \ + libgtest-dev \ + libre2-dev \ + libc-ares-dev \ + rapidjson-dev \ + python3 \ + python3-six \ + ragel \ + yasm + + cd /source + cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex + cmake --build build_googleapis_deb -j$(nproc) + cmake --build build_googleapis_deb --target package + dpkg -i build_googleapis_deb/*.deb + + ./scripts/generate-debian-directory.sh + cmake -S . -B build-deb \ + -DCMAKE_BUILD_TYPE=Release \ + -DYDB_SDK_INSTALL=ON \ + -DYDB_SDK_EXAMPLES=OFF \ + -DYDB_SDK_TESTS=OFF \ + -DYDB_SDK_ENABLE_OTEL_METRICS=ON \ + -DYDB_SDK_ENABLE_OTEL_TRACE=ON \ + -DBUILD_SHARED_LIBS=OFF \ + -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ + -DCMAKE_PREFIX_PATH="/usr/share/yandex" + cmake --build build-deb --target package -j$(nproc) + + cp build_googleapis_deb/*.deb /source/artifacts/ + cp build-deb/*.deb /source/artifacts/ + ' + + - name: Smoke-test generated .deb packages + shell: bash + run: | + ./scripts/test_deb_packages.sh artifacts + + - name: List built artifacts + shell: bash + run: ls -la artifacts/ + + - name: Upload .deb files to GitHub release + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + gh release upload "${{ steps.tag.outputs.tag }}" artifacts/*.deb --clobber + + - name: Upload .deb files as workflow artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: deb-packages-${{ steps.tag.outputs.tag }} + path: artifacts/*.deb + if-no-files-found: warn + retention-days: 30 diff --git a/README.md b/README.md index eb9aadb503a..564c69b1fc8 100644 --- a/README.md +++ b/README.md @@ -191,31 +191,29 @@ To smoke-test generated `.deb` packages with the sample consumer project: ./scripts/test_deb_packages.sh build-deb ``` -### Install from PPA +### Install from GitHub releases -Pre-built packages are available from the Launchpad PPA. This is the easiest -way to install the SDK on Ubuntu 24.04 (Noble): +Pre-built `.deb` packages for Ubuntu 24.04 (Noble) are attached to each +GitHub release. Download the assets and install them with `dpkg`: ```bash -sudo add-apt-repository ppa:ydb-team/ydb-cpp-sdk -sudo apt-get update -sudo apt-get install libydb-cpp-dev +# Replace with the desired release tag (e.g. v1.2.3) +TAG= +BASE="https://github.com/ydb-platform/ydb-cpp-sdk/releases/download/${TAG}" + +wget "${BASE}/yandex-googleapis-api-common-protos_1.0.0_amd64.deb" +wget "${BASE}/libydb-cpp-dev_${TAG#v}_amd64.deb" +# Optional plugins: +wget "${BASE}/libydb-cpp-iam-dev_${TAG#v}_amd64.deb" +wget "${BASE}/libydb-cpp-otel-metrics-dev_${TAG#v}_amd64.deb" +wget "${BASE}/libydb-cpp-otel-tracing-dev_${TAG#v}_amd64.deb" + +sudo dpkg -i yandex-googleapis-api-common-protos_*.deb +sudo dpkg -i libydb-cpp-dev_*.deb libydb-cpp-iam-dev_*.deb \ + libydb-cpp-otel-metrics-dev_*.deb libydb-cpp-otel-tracing-dev_*.deb +sudo apt-get install -f # resolve any remaining dependencies ``` -Optional packages: - -```bash -# IAM credentials plugin -sudo apt-get install libydb-cpp-iam-dev - -# OpenTelemetry plugins -sudo apt-get install libydb-cpp-otel-metrics-dev libydb-cpp-otel-tracing-dev -``` - -The PPA also provides `yandex-googleapis-api-common-protos` and -`yandex-opentelemetry-cpp-dev` which are installed automatically as -dependencies. - After installation, use the SDK in your CMake project: ```cmake @@ -223,25 +221,9 @@ find_package(ydb-cpp-sdk REQUIRED COMPONENTS Driver Table Topic) target_link_libraries(myapp PRIVATE YDB-CPP-SDK::Driver YDB-CPP-SDK::Table) ``` -Pass `-DCMAKE_PREFIX_PATH=/usr/share/yandex` if the packages are installed +Pass `-DCMAKE_PREFIX_PATH=/usr/share/yandex` since the packages install under the Yandex prefix. -### Publish to PPA - -To upload new package versions to the PPA (maintainers only): - -```bash -# Dry run — build source packages without uploading -./scripts/upload_to_ppa.sh --all --dry-run --series noble - -# Actual upload (requires GPG key registered on Launchpad) -./scripts/upload_to_ppa.sh --all --gpg-key --series noble -``` - -Packages must be uploaded in order: `googleapis` → `otel` → `sdk`. -See [plans/ppa-setup.md](plans/ppa-setup.md) for detailed PPA creation -and GPG key setup instructions. - ### Test Specify a level of parallelism by passing the `-j` option into the command below (e.g. `-j$(nproc)`) diff --git a/plans/ppa-setup.md b/plans/ppa-setup.md deleted file mode 100644 index e497d82076a..00000000000 --- a/plans/ppa-setup.md +++ /dev/null @@ -1,275 +0,0 @@ -# PPA Setup Guide for YDB C++ SDK - -This document describes how to create and configure a Launchpad PPA for -distributing YDB C++ SDK Debian packages. - -## Table of Contents - -1. [Prerequisites](#prerequisites) -2. [GPG Key Generation](#gpg-key-generation) -3. [Launchpad Account Setup](#launchpad-account-setup) -4. [PPA Creation](#ppa-creation) -5. [Local Configuration](#local-configuration) -6. [GitHub Secrets Configuration](#github-secrets-configuration) -7. [Testing the Upload Flow](#testing-the-upload-flow) -8. [Package Upload Order](#package-upload-order) -9. [Version Bumping](#version-bumping) -10. [Troubleshooting](#troubleshooting) - ---- - -## Prerequisites - -Install the required tools on your development machine (Ubuntu 22.04+): - -```bash -sudo apt-get update -sudo apt-get install -y \ - devscripts \ - dput \ - debhelper \ - dpkg-dev \ - gnupg \ - git -``` - -## GPG Key Generation - -Launchpad requires a GPG key to sign source packages. - -### Generate a new key - -```bash -gpg --full-generate-key -``` - -When prompted: -- **Key type**: RSA and RSA (default) -- **Key size**: 4096 bits -- **Expiration**: 0 (does not expire) — or set a reasonable expiration -- **Real name**: `YDB Team` (or your name) -- **Email**: `ydb-team@ydb.tech` (must match your Launchpad account email) -- **Passphrase**: choose a strong passphrase - -### Find your key ID - -```bash -gpg --list-secret-keys --keyid-format long -``` - -Output example: -``` -sec rsa4096/ABCDEF1234567890 2024-01-01 [SC] - FINGERPRINT1234567890ABCDEF1234567890ABCDEF -uid [ultimate] YDB Team -ssb rsa4096/1234567890ABCDEF 2024-01-01 [E] -``` - -Your **key ID** is `ABCDEF1234567890` (the part after `rsa4096/`). -Your **fingerprint** is the full 40-character hex string. - -### Upload key to Ubuntu keyserver - -```bash -gpg --keyserver keyserver.ubuntu.com --send-keys ABCDEF1234567890 -``` - -Wait a few minutes for propagation. - -### Export the private key (for CI) - -```bash -gpg --armor --export-secret-keys ABCDEF1234567890 > gpg-private-key.asc -``` - -**Keep this file secure!** It will be stored as a GitHub secret. - -## Launchpad Account Setup - -1. Go to https://launchpad.net and create an account (or sign in). -2. Navigate to https://launchpad.net/~/+editpgpkeys -3. Paste your GPG key **fingerprint** (40-character hex string). -4. Launchpad will send an encrypted email to verify ownership. -5. Decrypt the email and follow the confirmation link. - -## PPA Creation - -1. Go to https://launchpad.net/~/+activate-ppa -2. Fill in: - - **URL**: `ydb-cpp-sdk` - - **Display name**: `YDB C++ SDK` - - **Description**: - ``` - YDB C++ SDK development packages and dependencies. - - Packages: - - yandex-googleapis-api-common-protos: compiled Google API common proto specs - - yandex-opentelemetry-cpp-dev: OpenTelemetry C++ SDK - - libydb-cpp-dev: YDB C++ SDK core library - - libydb-cpp-iam-dev: YDB C++ SDK IAM plugin - - libydb-cpp-otel-metrics-dev: YDB C++ SDK OpenTelemetry metrics plugin - - libydb-cpp-otel-tracing-dev: YDB C++ SDK OpenTelemetry tracing plugin - ``` -3. Click **Activate**. - -The PPA URL will be: `ppa:YOUR_LAUNCHPAD_USERNAME/ydb-cpp-sdk` - -### Enable additional architectures (optional) - -By default, PPAs build for `amd64` and `i386`. To add `arm64`: - -1. Go to your PPA page on Launchpad -2. Click **Change details** -3. Under **Processors**, check `ARM ARMv8 (arm64)` - -## Local Configuration - -### Configure dput - -Create or edit `~/.dput.cf`: - -```ini -[ydb-cpp-sdk] -fqdn = ppa.launchpad.net -method = ftp -incoming = ~YOUR_LAUNCHPAD_USERNAME/ubuntu/ydb-cpp-sdk/ -login = anonymous -allow_unsigned_uploads = 0 -``` - -Alternatively, use the shorthand PPA syntax with `dput`: -```bash -dput ppa:YOUR_LAUNCHPAD_USERNAME/ydb-cpp-sdk -``` - -## GitHub Secrets Configuration - -For automated CI publishing, add these secrets to your GitHub repository -(Settings → Secrets and variables → Actions): - -| Secret Name | Description | How to obtain | -|---|---|---| -| `GPG_PRIVATE_KEY` | ASCII-armored GPG private key | `gpg --armor --export-secret-keys KEY_ID` | -| `GPG_PASSPHRASE` | Passphrase for the GPG key | The passphrase you chose during key generation | -| `LAUNCHPAD_PPA` | PPA identifier | `ppa:YOUR_USERNAME/ydb-cpp-sdk` | - -### Setting secrets via GitHub CLI - -```bash -gh secret set GPG_PRIVATE_KEY < gpg-private-key.asc -gh secret set GPG_PASSPHRASE --body "your-passphrase" -gh secret set LAUNCHPAD_PPA --body "ppa:ydb-team/ydb-cpp-sdk" -``` - -## Testing the Upload Flow - -### Dry run (no upload) - -```bash -# Build all source packages without uploading -./scripts/upload_to_ppa.sh --all --dry-run --series noble - -# Build only googleapis -./scripts/upload_to_ppa.sh --googleapis --dry-run - -# Build only SDK -./scripts/upload_to_ppa.sh --sdk --dry-run --series noble -``` - -### Actual upload - -```bash -# Upload all packages -./scripts/upload_to_ppa.sh --all --gpg-key ABCDEF1234567890 --series noble - -# Upload only googleapis first (required before SDK) -./scripts/upload_to_ppa.sh --googleapis --gpg-key ABCDEF1234567890 - -# Then upload opentelemetry-cpp -./scripts/upload_to_ppa.sh --otel --gpg-key ABCDEF1234567890 - -# Then upload SDK (after googleapis and otel are published) -./scripts/upload_to_ppa.sh --sdk --gpg-key ABCDEF1234567890 -``` - -### Via GitHub Actions - -1. Go to **Actions** → **Publish to PPA** -2. Click **Run workflow** -3. Select parameters: - - Series: `noble` - - Packages: `all` - - Dry run: `true` (for testing) -4. Review the output -5. Re-run with dry_run=`false` for actual upload - -## Package Upload Order - -**Important**: Packages must be uploaded and **published** in this order -because of build dependencies: - -1. `yandex-googleapis-api-common-protos` — no PPA dependencies -2. `yandex-opentelemetry-cpp` — no PPA dependencies -3. `ydb-cpp-sdk` — depends on both (1) and (2) at build time - -Wait for each package to be **published** on Launchpad before uploading -the next one. Launchpad build times vary but typically take 15–60 minutes. - -You can check build status at: -`https://launchpad.net/~YOUR_USERNAME/+archive/ubuntu/ydb-cpp-sdk/+packages` - -## Version Bumping - -### Rules for PPA versions - -- Each upload to the same series must have a **unique version**. -- Use the `~SERIES1` suffix convention: `3.18.0-1~noble1` -- For rebuilds, increment the suffix: `3.18.0-1~noble2`, `3.18.0-1~noble3` -- For multiple series, use different suffixes: `3.18.0-1~noble1`, `3.18.0-1~jammy1` - -### Updating the SDK version - -1. Edit `src/version.h` and update `YDB_SDK_VERSION` -2. The `generate-debian-directory.sh` script automatically extracts the version -3. Run `./scripts/upload_to_ppa.sh --sdk --series noble` - -### Updating the googleapis version - -1. Update the `CPACK_PACKAGE_VERSION` in `scripts/googleapis_deb/CMakeLists.txt` -2. Run `./scripts/upload_to_ppa.sh --googleapis --series noble` - -## Troubleshooting - -### "Signature verification failed" - -- Ensure your GPG key is uploaded to `keyserver.ubuntu.com` -- Ensure the key is registered on your Launchpad account -- Wait a few minutes after uploading the key for propagation - -### "Version already exists" - -- Increment the PPA suffix (e.g., `~noble1` → `~noble2`) -- Or delete the existing package from the PPA web interface - -### "Build dependency not satisfiable" - -- Ensure prerequisite packages are published in the PPA first -- Check that package names in `debian/control` match exactly - -### "Source format 3.0 (quilt) requires .orig.tar.gz" - -- Run without `--skip-orig` flag -- Or ensure the `.orig.tar.gz` was previously uploaded for this version - -### Build fails on Launchpad - -- Check the build log on the Launchpad PPA page -- Common issues: missing Build-Depends, network access during build - (Launchpad builds are sandboxed — no internet access) -- For opentelemetry-cpp: the FetchContent approach won't work on Launchpad; - the source must be included in the orig tarball - -### "Rejected: Could not find person" - -- The PPA identifier is wrong. Use `ppa:USERNAME/PPA_NAME` format. -- Verify with: `https://launchpad.net/~USERNAME/+archive/ubuntu/PPA_NAME` diff --git a/plans/ppa-upload-plan.md b/plans/ppa-upload-plan.md deleted file mode 100644 index 6fa53ac66d2..00000000000 --- a/plans/ppa-upload-plan.md +++ /dev/null @@ -1,266 +0,0 @@ -# Plan: PPA Upload Script & Launchpad PPA Setup - -## Overview - -This plan covers two deliverables: -1. A script (`scripts/upload_to_ppa.sh`) that builds Debian **source packages** and uploads them to a Launchpad PPA. -2. Complete instructions for creating and configuring the PPA on Launchpad. - -The PPA will host **two source packages**: -- `yandex-googleapis-api-common-protos` — compiled Google API common proto specs (static libs, `.pb.h`, `.pb.cc`, Python stubs) -- `ydb-cpp-sdk` — the SDK itself (4 binary packages: `libydb-cpp-dev`, `libydb-cpp-iam-dev`, `libydb-cpp-otel-metrics-dev`, `libydb-cpp-otel-tracing-dev`) - -## Architecture - -```mermaid -flowchart TD - A[Developer runs scripts/upload_to_ppa.sh] --> B{Which packages?} - B -->|--googleapis| C[Build googleapis source package] - B -->|--sdk| D[Build ydb-cpp-sdk source package] - B -->|--all or default| C - B -->|--all or default| D - - C --> C1[scripts/googleapis_deb/generate-debian-directory.sh] - C1 --> C2[debuild -S -sa] - C2 --> C3[dput ppa:ydb-team/ydb-cpp-sdk ..._source.changes] - - D --> D1[scripts/generate-debian-directory.sh] - D1 --> D2[debuild -S -sa] - D2 --> D3[dput ppa:ydb-team/ydb-cpp-sdk ..._source.changes] - - C3 --> E[Launchpad PPA builds .deb packages] - D3 --> E - - E --> F[Users: sudo add-apt-repository ppa:ydb-team/ydb-cpp-sdk] - F --> G[sudo apt install libydb-cpp-dev] -``` - -## Key Concepts: How Launchpad PPAs Work - -1. **PPAs only accept source packages** — Launchpad builds the `.deb` binaries on its own build farm. -2. Source packages consist of: `_.orig.tar.gz`, `_.debian.tar.xz`, `_.dsc`, `__source.changes`. -3. The `.changes` file must be **GPG-signed** with a key registered on Launchpad. -4. Upload is done via `dput` to `ppa:OWNER/PPA_NAME`. -5. Each source package needs a proper `debian/` directory with `changelog`, `control`, `rules`, `source/format`, and optionally `.install` files. - -## Detailed Steps - -### 1. Create `scripts/googleapis_deb/generate-debian-directory.sh` - -This script generates `debian/` metadata for the `yandex-googleapis-api-common-protos` source package. It must: - -- Extract version from `scripts/googleapis_deb/CMakeLists.txt` (currently `1.0.0`) -- Generate `debian/changelog` with proper format for the target Ubuntu series (e.g., `noble`) -- Generate `debian/control` with: - - `Source: yandex-googleapis-api-common-protos` - - `Build-Depends: debhelper-compat (= 13), cmake, protobuf-compiler, libprotobuf-dev` - - Binary package: `yandex-googleapis-api-common-protos` - - `Depends: libprotobuf-dev` -- Generate `debian/rules` using cmake buildsystem with `CMAKE_INSTALL_PREFIX=/usr/share/yandex` -- Generate `debian/source/format` as `3.0 (quilt)` -- Generate `.install` file mapping built artifacts - -**Key detail**: The CMakeLists.txt at `scripts/googleapis_deb/CMakeLists.txt` references `../../third_party/api-common-protos` — for the source package, the proto files must be included in the orig tarball. The script must handle this by creating a proper source tree. - -### 2. Update `scripts/generate-debian-directory.sh` - -The existing script already generates proper `debian/` metadata for the SDK. Minor updates needed: - -- Add `--series` parameter to target specific Ubuntu release (default: `noble` for 24.04) -- Ensure the changelog entry targets the correct series name instead of `unstable` -- Add `yandex-googleapis-api-common-protos` as a Build-Depends and runtime Depends for `libydb-cpp-dev` - -### 3. Create `scripts/upload_to_ppa.sh` - -Main upload script with the following interface: - -```bash -./scripts/upload_to_ppa.sh [OPTIONS] - -Options: - --ppa PPA_NAME PPA identifier (default: ppa:ydb-team/ydb-cpp-sdk) - --series SERIES Ubuntu series (default: noble) - --googleapis Build and upload only googleapis package - --sdk Build and upload only SDK packages - --all Build and upload all packages (default) - --gpg-key KEY_ID GPG key ID for signing - --dry-run Build source packages but do not upload - --skip-orig Skip creating .orig.tar.gz (use -sd instead of -sa) -``` - -The script will: - -1. **Validate prerequisites**: check for `debuild`, `dput`, `gpg`, `git` -2. **For googleapis package**: - - Create a temporary build directory - - Copy `scripts/googleapis_deb/` and `third_party/api-common-protos/` into it - - Run `scripts/googleapis_deb/generate-debian-directory.sh` - - Create `.orig.tar.gz` from the source tree - - Run `debuild -S -sa` (or `-sd` if `--skip-orig`) - - Run `dput $PPA ..._source.changes` -3. **For SDK package**: - - Run `scripts/generate-debian-directory.sh --series $SERIES` - - Create `.orig.tar.gz` from the git archive (excluding `.git`, build dirs) - - Run `debuild -S -sa` - - Run `dput $PPA ..._source.changes` - -### 4. Create `.github/workflows/ppa_publish.yaml` - -GitHub Actions workflow for automated PPA publishing on release tags: - -```yaml -on: - release: - types: [published] - workflow_dispatch: - inputs: - series: - description: Ubuntu series - default: noble - dry_run: - description: Dry run - type: boolean - default: true -``` - -The workflow will: -- Import GPG private key from GitHub secret `GPG_PRIVATE_KEY` -- Configure `dput` with PPA settings -- Run `scripts/upload_to_ppa.sh --all --series $SERIES --gpg-key $KEY_ID` -- Use `workflow_dispatch` for manual triggers with dry-run option - -### 5. Update `README.md` - -Add sections: -- **Using the PPA** — instructions for end users to add the PPA and install packages -- **Publishing to PPA** — instructions for maintainers on GPG setup and upload process - -### 6. Create `plans/ppa-setup.md` - -Comprehensive PPA setup documentation covering: -- Launchpad account creation -- GPG key generation and registration -- PPA creation on Launchpad -- GitHub secrets configuration -- Testing the upload flow - -## File Changes Summary - -| File | Action | Description | -|------|--------|-------------| -| `scripts/upload_to_ppa.sh` | **Create** | Main PPA upload script | -| `scripts/googleapis_deb/generate-debian-directory.sh` | **Create** | Debian metadata generator for googleapis package | -| `scripts/generate-debian-directory.sh` | **Modify** | Add --series param, fix changelog series | -| `.github/workflows/ppa_publish.yaml` | **Create** | CI workflow for automated PPA publishing | -| `README.md` | **Modify** | Add PPA usage and publishing instructions | -| `plans/ppa-setup.md` | **Create** | Detailed PPA setup documentation | - -## PPA Creation Instructions (for `plans/ppa-setup.md`) - -### Prerequisites - -1. **Launchpad account** at https://launchpad.net -2. **GPG key** (RSA 4096-bit recommended) -3. **Ubuntu development tools**: `sudo apt install devscripts dput debhelper gpg` - -### Step-by-step PPA Setup - -#### 1. Generate GPG Key - -```bash -gpg --full-generate-key -# Choose: RSA and RSA, 4096 bits, no expiration -# Name: YDB Team -# Email: -``` - -#### 2. Upload GPG Key to Ubuntu Keyserver - -```bash -gpg --keyserver keyserver.ubuntu.com --send-keys -``` - -#### 3. Register GPG Key on Launchpad - -- Go to https://launchpad.net/~/+editpgpkeys -- Paste the GPG key fingerprint -- Confirm via email - -#### 4. Create PPA on Launchpad - -- Go to https://launchpad.net/~/+activate-ppa -- Name: `ydb-cpp-sdk` -- Display name: `YDB C++ SDK` -- Description: `YDB C++ SDK development packages and dependencies` -- Distribution: `Ubuntu` -- Architectures: `amd64`, `arm64` (optional) - -#### 5. Configure `~/.dput.cf` - -```ini -[ydb-cpp-sdk] -fqdn = ppa.launchpad.net -method = ftp -incoming = ~ydb-team/ubuntu/ydb-cpp-sdk/ -login = anonymous -allow_unsigned_uploads = 0 -``` - -Or use the shorthand: `dput ppa:ydb-team/ydb-cpp-sdk ` - -#### 6. Configure GitHub Secrets - -For CI automation, add these secrets to the GitHub repository: - -| Secret | Description | -|--------|-------------| -| `GPG_PRIVATE_KEY` | ASCII-armored GPG private key (`gpg --armor --export-secret-keys KEY_ID`) | -| `GPG_PASSPHRASE` | Passphrase for the GPG key | -| `LAUNCHPAD_PPA` | PPA identifier (e.g., `ppa:ydb-team/ydb-cpp-sdk`) | - -#### 7. Test Upload - -```bash -# Dry run (builds source packages without uploading) -./scripts/upload_to_ppa.sh --all --dry-run - -# Actual upload -./scripts/upload_to_ppa.sh --all --gpg-key -``` - -#### 8. End-User Installation - -Once packages are published: - -```bash -sudo add-apt-repository ppa:ydb-team/ydb-cpp-sdk -sudo apt update -sudo apt install libydb-cpp-dev libydb-cpp-iam-dev -``` - -## Package Dependency Chain - -```mermaid -flowchart LR - A[yandex-googleapis-api-common-protos] --> B[libydb-cpp-dev] - B --> C[libydb-cpp-iam-dev] - B --> D[libydb-cpp-otel-metrics-dev] - B --> E[libydb-cpp-otel-tracing-dev] - - F[libprotobuf-dev] --> A - F --> B - G[libgrpc++-dev] --> B - H[opentelemetry-cpp] --> D - H --> E -``` - -## Important Notes - -1. **PPA builds happen on Launchpad servers** — all Build-Depends must be available in the target Ubuntu release or in the same PPA. -2. **googleapis package must be uploaded first** — the SDK depends on it at build time. -3. **OpenTelemetry**: Ubuntu 24.04 does not ship `opentelemetry-cpp` packages. Options: - - Build and upload an `opentelemetry-cpp` package to the same PPA - - Disable OTel plugins in the PPA build (simpler, but loses functionality) - - Use vendored/FetchContent approach in the PPA build -4. **Version bumping**: Each upload to the same series must have a unique version. Use `~ppa1`, `~ppa2` suffixes for rebuilds. -5. **The `CPACK_PACKAGING_INSTALL_PREFIX`** in `scripts/googleapis_deb/CMakeLists.txt` is set to `/usr/share/yandex` — this must match the `CMAKE_INSTALL_PREFIX` and `CMAKE_PREFIX_PATH` used by the SDK build. diff --git a/scripts/googleapis_deb/generate-debian-directory.sh b/scripts/googleapis_deb/generate-debian-directory.sh deleted file mode 100755 index 5b6a10f22ff..00000000000 --- a/scripts/googleapis_deb/generate-debian-directory.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash -# Generate debian/ directory for yandex-googleapis-api-common-protos source package. -# This script is intended to be run from the repository root or from -# scripts/googleapis_deb/ itself. It produces a debian/ directory inside -# scripts/googleapis_deb/ that is suitable for dpkg-buildpackage / debuild. -set -euo pipefail - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -REPO_ROOT=$(realpath "$SCRIPT_DIR/../..") - -SERIES="${1:-noble}" - -# --------------------------------------------------------------------------- -# Version -# --------------------------------------------------------------------------- -VERSION=$(grep -oP 'CPACK_PACKAGE_VERSION\s+"\K[^"]+' "$SCRIPT_DIR/CMakeLists.txt" || echo "1.0.0") -GIT_COMMIT=$(cd "$REPO_ROOT" && git rev-parse --short HEAD 2>/dev/null || echo "unknown") -DEB_VERSION="${VERSION}-1~${SERIES}1" - -# --------------------------------------------------------------------------- -# debian/ skeleton -# --------------------------------------------------------------------------- -DEB_DIR="$SCRIPT_DIR/debian" -mkdir -p "$DEB_DIR/source" - -# -- changelog -------------------------------------------------------------- -cat > "$DEB_DIR/changelog" < $(date -R) -EOF - -# -- control ---------------------------------------------------------------- -cat > "$DEB_DIR/control" <<'EOF' -Source: yandex-googleapis-api-common-protos -Section: libdevel -Priority: optional -Maintainer: YDB Team -Build-Depends: debhelper-compat (= 13), - cmake, - protobuf-compiler, - libprotobuf-dev -Standards-Version: 4.6.2 -Homepage: https://github.com/googleapis/api-common-protos -Rules-Requires-Root: no - -Package: yandex-googleapis-api-common-protos -Architecture: any -Depends: ${misc:Depends}, libprotobuf-dev -Description: Google API common proto compiled libraries (Yandex build) - Pre-compiled C++ static libraries and headers generated from the - googleapis/api-common-protos proto definitions. Installed under - /usr/share/yandex so that YDB C++ SDK and other Yandex packages can - find_package(yandex-googleapis-api-common-protos). -EOF - -# -- rules ------------------------------------------------------------------ -cat > "$DEB_DIR/rules" <<'RULES' -#!/usr/bin/make -f - -export DEB_BUILD_MAINT_OPTIONS = hardening=+all - -%: - dh $@ --buildsystem=cmake - -override_dh_auto_configure: - dh_auto_configure -- \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex - -override_dh_auto_install: - dh_auto_install --destdir=debian/tmp -RULES -chmod +x "$DEB_DIR/rules" - -# -- install ---------------------------------------------------------------- -cat > "$DEB_DIR/yandex-googleapis-api-common-protos.install" <<'EOF' -debian/tmp/usr/share/yandex/lib/*/libapi-common-protos.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/include/google usr/share/yandex/include/ -debian/tmp/usr/share/yandex/lib/*/cmake/yandex-googleapis-api-common-protos usr/share/yandex/lib/cmake/ -EOF - -# -- source/format ---------------------------------------------------------- -echo "3.0 (quilt)" > "$DEB_DIR/source/format" - -echo "Generated debian directory for yandex-googleapis-api-common-protos ${DEB_VERSION} (series: ${SERIES})" diff --git a/scripts/otel_deb/CMakeLists.txt b/scripts/otel_deb/CMakeLists.txt deleted file mode 100644 index da848d2b3b7..00000000000 --- a/scripts/otel_deb/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -cmake_minimum_required(VERSION 3.14) -project(yandex-opentelemetry-cpp VERSION 1.12.0) - -set(CMAKE_CXX_STANDARD 20) - -include(FetchContent) - -set(BUILD_TESTING OFF CACHE BOOL "" FORCE) -set(WITH_OTLP_HTTP OFF CACHE BOOL "" FORCE) -set(WITH_OTLP_GRPC OFF CACHE BOOL "" FORCE) -set(WITH_PROMETHEUS OFF CACHE BOOL "" FORCE) -set(WITH_EXAMPLES OFF CACHE BOOL "" FORCE) -set(OTELCPP_MAINTAINER_MODE OFF CACHE BOOL "" FORCE) -set(CMAKE_POSITION_INDEPENDENT_CODE ON CACHE BOOL "" FORCE) -set(WITH_STL "CXX20" CACHE STRING "" FORCE) -set(OPENTELEMETRY_INSTALL ON CACHE BOOL "" FORCE) - -FetchContent_Declare( - opentelemetry-cpp - GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git - GIT_TAG v1.12.0 -) -FetchContent_MakeAvailable(opentelemetry-cpp) diff --git a/scripts/otel_deb/generate-debian-directory.sh b/scripts/otel_deb/generate-debian-directory.sh deleted file mode 100755 index cb96d28b0fd..00000000000 --- a/scripts/otel_deb/generate-debian-directory.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# Generate debian/ directory for yandex-opentelemetry-cpp source package. -# This wraps the upstream opentelemetry-cpp v1.12.0 into a Debian source -# package that installs under /usr/share/yandex so it can be consumed by -# the YDB C++ SDK packaging build. -set -euo pipefail - -SCRIPT_DIR=$(dirname "$(realpath "$0")") - -SERIES="${1:-noble}" -OTEL_VERSION="1.12.0" -GIT_COMMIT="v${OTEL_VERSION}" -DEB_VERSION="${OTEL_VERSION}-1~${SERIES}1" - -# --------------------------------------------------------------------------- -# debian/ skeleton -# --------------------------------------------------------------------------- -DEB_DIR="$SCRIPT_DIR/debian" -mkdir -p "$DEB_DIR/source" - -# -- changelog -------------------------------------------------------------- -cat > "$DEB_DIR/changelog" < $(date -R) -EOF - -# -- control ---------------------------------------------------------------- -cat > "$DEB_DIR/control" <<'EOF' -Source: yandex-opentelemetry-cpp -Section: libdevel -Priority: optional -Maintainer: YDB Team -Build-Depends: debhelper-compat (= 13), - cmake, - git, - pkg-config, - libcurl4-openssl-dev, - nlohmann-json3-dev -Standards-Version: 4.6.2 -Homepage: https://opentelemetry.io/docs/languages/cpp/ -Rules-Requires-Root: no - -Package: yandex-opentelemetry-cpp-dev -Architecture: any -Depends: ${misc:Depends} -Description: OpenTelemetry C++ SDK (Yandex build) - Static libraries, headers and CMake package files for the OpenTelemetry - C++ SDK. Installed under /usr/share/yandex so that YDB C++ SDK and - other Yandex packages can find_package(opentelemetry-cpp). -EOF - -# -- rules ------------------------------------------------------------------ -cat > "$DEB_DIR/rules" <<'RULES' -#!/usr/bin/make -f - -export DEB_BUILD_MAINT_OPTIONS = hardening=+all - -%: - dh $@ --buildsystem=cmake - -override_dh_auto_configure: - dh_auto_configure -- \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \ - -DFETCHCONTENT_FULLY_DISCONNECTED=OFF - -override_dh_auto_install: - dh_auto_install --destdir=debian/tmp -RULES -chmod +x "$DEB_DIR/rules" - -# -- install ---------------------------------------------------------------- -cat > "$DEB_DIR/yandex-opentelemetry-cpp-dev.install" <<'EOF' -debian/tmp/usr/share/yandex/lib/*/libopentelemetry_*.a usr/share/yandex/lib/ -debian/tmp/usr/share/yandex/include/opentelemetry usr/share/yandex/include/ -debian/tmp/usr/share/yandex/lib/*/cmake/opentelemetry-cpp usr/share/yandex/lib/cmake/ -debian/tmp/usr/share/yandex/lib/*/pkgconfig/opentelemetry_*.pc usr/share/yandex/lib/pkgconfig/ -EOF - -# -- source/format ---------------------------------------------------------- -echo "3.0 (quilt)" > "$DEB_DIR/source/format" - -echo "Generated debian directory for yandex-opentelemetry-cpp ${DEB_VERSION} (series: ${SERIES})" diff --git a/scripts/upload_to_ppa.sh b/scripts/upload_to_ppa.sh deleted file mode 100755 index be4a3ac2722..00000000000 --- a/scripts/upload_to_ppa.sh +++ /dev/null @@ -1,300 +0,0 @@ -#!/bin/bash -# ============================================================================= -# upload_to_ppa.sh — Build Debian source packages and upload them to a -# Launchpad PPA. -# -# Usage: -# ./scripts/upload_to_ppa.sh [OPTIONS] -# -# Options: -# --ppa PPA PPA identifier (default: ppa:ydb-team/ydb-cpp-sdk) -# --series SERIES Ubuntu series name (default: noble) -# --googleapis Build/upload only the googleapis package -# --otel Build/upload only the opentelemetry-cpp package -# --sdk Build/upload only the ydb-cpp-sdk packages -# --all Build/upload all packages (default) -# --gpg-key KEY_ID GPG key fingerprint for signing -# --dry-run Build source packages but do not upload -# --skip-orig Pass -sd to debuild (reuse existing .orig.tar.gz) -# --help Show this help message -# ============================================================================= -set -euo pipefail - -SCRIPT_DIR=$(dirname "$(realpath "$0")") -REPO_ROOT=$(realpath "$SCRIPT_DIR/..") - -# ---- defaults --------------------------------------------------------------- -PPA="ppa:ydb-team/ydb-cpp-sdk" -SERIES="noble" -GPG_KEY="" -DRY_RUN=false -SKIP_ORIG=false -BUILD_GOOGLEAPIS=false -BUILD_OTEL=false -BUILD_SDK=false -BUILD_ALL=true - -# ---- parse arguments -------------------------------------------------------- -while [[ $# -gt 0 ]]; do - case "$1" in - --ppa) PPA="$2"; shift 2 ;; - --ppa=*) PPA="${1#*=}"; shift ;; - --series) SERIES="$2"; shift 2 ;; - --series=*) SERIES="${1#*=}"; shift ;; - --gpg-key) GPG_KEY="$2"; shift 2 ;; - --gpg-key=*) GPG_KEY="${1#*=}"; shift ;; - --googleapis) BUILD_GOOGLEAPIS=true; BUILD_ALL=false; shift ;; - --otel) BUILD_OTEL=true; BUILD_ALL=false; shift ;; - --sdk) BUILD_SDK=true; BUILD_ALL=false; shift ;; - --all) BUILD_ALL=true; shift ;; - --dry-run) DRY_RUN=true; shift ;; - --skip-orig) SKIP_ORIG=true; shift ;; - --help) - head -n 20 "$0" | grep '^#' | sed 's/^# \?//' - exit 0 - ;; - *) - echo "Error: unknown option '$1'" >&2 - echo "Run $0 --help for usage." >&2 - exit 1 - ;; - esac -done - -if $BUILD_ALL; then - BUILD_GOOGLEAPIS=true - BUILD_OTEL=true - BUILD_SDK=true -fi - -# ---- prerequisites ---------------------------------------------------------- -for cmd in debuild dput gpg git tar; do - if ! command -v "$cmd" &>/dev/null; then - echo "Error: '$cmd' is not installed. Install it first:" >&2 - echo " sudo apt install devscripts dput gnupg git" >&2 - exit 1 - fi -done - -DEBUILD_SIGN_ARGS=() -if [[ -n "$GPG_KEY" ]]; then - DEBUILD_SIGN_ARGS=(-k"$GPG_KEY") -fi - -DEBUILD_ORIG_FLAG="-sa" -if $SKIP_ORIG; then - DEBUILD_ORIG_FLAG="-sd" -fi - -# ---- helper: upload or dry-run ---------------------------------------------- -do_upload() { - local changes_file="$1" - if $DRY_RUN; then - echo "[dry-run] Would upload: $changes_file" - echo "[dry-run] dput $PPA $changes_file" - else - echo "Uploading $changes_file to $PPA ..." - dput "$PPA" "$changes_file" - fi -} - -# ============================================================================= -# 1. yandex-googleapis-api-common-protos -# ============================================================================= -if $BUILD_GOOGLEAPIS; then - echo "============================================================" - echo "Building source package: yandex-googleapis-api-common-protos" - echo "============================================================" - - GAPI_SRC="$SCRIPT_DIR/googleapis_deb" - GAPI_VERSION=$(grep -oP 'CPACK_PACKAGE_VERSION\s+"\K[^"]+' "$GAPI_SRC/CMakeLists.txt" || echo "1.0.0") - GAPI_DEB_VERSION="${GAPI_VERSION}-1~${SERIES}1" - GAPI_PKG="yandex-googleapis-api-common-protos" - - WORK_DIR=$(mktemp -d) - trap "rm -rf $WORK_DIR" EXIT - - GAPI_BUILD_DIR="$WORK_DIR/${GAPI_PKG}-${GAPI_VERSION}" - mkdir -p "$GAPI_BUILD_DIR" - - # Copy CMakeLists.txt and config template - cp "$GAPI_SRC/CMakeLists.txt" "$GAPI_BUILD_DIR/" - cp "$GAPI_SRC/yandex-googleapis-api-common-protosConfig.cmake.in" "$GAPI_BUILD_DIR/" - - # Copy proto sources (the submodule content) - if [[ -d "$REPO_ROOT/third_party/api-common-protos/google" ]]; then - mkdir -p "$GAPI_BUILD_DIR/third_party/api-common-protos" - cp -r "$REPO_ROOT/third_party/api-common-protos/google" \ - "$GAPI_BUILD_DIR/third_party/api-common-protos/" - else - echo "Error: third_party/api-common-protos/google not found." >&2 - echo "Initialize git submodules: git submodule update --init" >&2 - exit 1 - fi - - # Fix CMakeLists.txt PROTOS_DIR to point to local copy - sed -i 's|"\${CMAKE_CURRENT_SOURCE_DIR}/../../third_party/api-common-protos"|"${CMAKE_CURRENT_SOURCE_DIR}/third_party/api-common-protos"|' \ - "$GAPI_BUILD_DIR/CMakeLists.txt" - - # Generate debian/ directory - bash "$GAPI_SRC/generate-debian-directory.sh" "$SERIES" - cp -r "$GAPI_SRC/debian" "$GAPI_BUILD_DIR/" - - # Create .orig.tar.gz - ORIG_TAR="$WORK_DIR/${GAPI_PKG}_${GAPI_VERSION}.orig.tar.gz" - tar -czf "$ORIG_TAR" -C "$WORK_DIR" "${GAPI_PKG}-${GAPI_VERSION}" - - # Build source package - cd "$GAPI_BUILD_DIR" - debuild -S "$DEBUILD_ORIG_FLAG" "${DEBUILD_SIGN_ARGS[@]}" - - # Upload - CHANGES_FILE="$WORK_DIR/${GAPI_PKG}_${GAPI_DEB_VERSION}_source.changes" - if [[ -f "$CHANGES_FILE" ]]; then - do_upload "$CHANGES_FILE" - else - echo "Warning: expected changes file not found: $CHANGES_FILE" >&2 - # Try to find it - CHANGES_FILE=$(ls "$WORK_DIR"/${GAPI_PKG}_*_source.changes 2>/dev/null | head -1) - if [[ -n "$CHANGES_FILE" ]]; then - do_upload "$CHANGES_FILE" - else - echo "Error: no _source.changes file found for googleapis." >&2 - exit 1 - fi - fi - - cd "$REPO_ROOT" - echo "googleapis source package built successfully." - echo "" -fi - -# ============================================================================= -# 2. yandex-opentelemetry-cpp -# ============================================================================= -if $BUILD_OTEL; then - echo "============================================================" - echo "Building source package: yandex-opentelemetry-cpp" - echo "============================================================" - - OTEL_SRC="$SCRIPT_DIR/otel_deb" - OTEL_VERSION="1.12.0" - OTEL_DEB_VERSION="${OTEL_VERSION}-1~${SERIES}1" - OTEL_PKG="yandex-opentelemetry-cpp" - - WORK_DIR_OTEL=$(mktemp -d) - trap "rm -rf $WORK_DIR_OTEL" EXIT - - OTEL_BUILD_DIR="$WORK_DIR_OTEL/${OTEL_PKG}-${OTEL_VERSION}" - mkdir -p "$OTEL_BUILD_DIR" - - # Copy CMakeLists.txt - cp "$OTEL_SRC/CMakeLists.txt" "$OTEL_BUILD_DIR/" - - # Generate debian/ directory - bash "$OTEL_SRC/generate-debian-directory.sh" "$SERIES" - cp -r "$OTEL_SRC/debian" "$OTEL_BUILD_DIR/" - - # Create .orig.tar.gz - ORIG_TAR_OTEL="$WORK_DIR_OTEL/${OTEL_PKG}_${OTEL_VERSION}.orig.tar.gz" - tar -czf "$ORIG_TAR_OTEL" -C "$WORK_DIR_OTEL" "${OTEL_PKG}-${OTEL_VERSION}" - - # Build source package - cd "$OTEL_BUILD_DIR" - debuild -S "$DEBUILD_ORIG_FLAG" "${DEBUILD_SIGN_ARGS[@]}" - - # Upload - CHANGES_FILE_OTEL="$WORK_DIR_OTEL/${OTEL_PKG}_${OTEL_DEB_VERSION}_source.changes" - if [[ -f "$CHANGES_FILE_OTEL" ]]; then - do_upload "$CHANGES_FILE_OTEL" - else - CHANGES_FILE_OTEL=$(ls "$WORK_DIR_OTEL"/${OTEL_PKG}_*_source.changes 2>/dev/null | head -1) - if [[ -n "$CHANGES_FILE_OTEL" ]]; then - do_upload "$CHANGES_FILE_OTEL" - else - echo "Error: no _source.changes file found for opentelemetry-cpp." >&2 - exit 1 - fi - fi - - cd "$REPO_ROOT" - echo "opentelemetry-cpp source package built successfully." - echo "" -fi - -# ============================================================================= -# 3. ydb-cpp-sdk -# ============================================================================= -if $BUILD_SDK; then - echo "============================================================" - echo "Building source package: ydb-cpp-sdk" - echo "============================================================" - - SDK_VERSION=$(grep -E 'YDB_SDK_VERSION = "[0-9]+\.[0-9]+\.[0-9]+"' "$REPO_ROOT/src/version.h" \ - | sed -E 's/.*"([0-9]+\.[0-9]+\.[0-9]+)".*/\1/') - if [[ -z "$SDK_VERSION" ]]; then - echo "Error: Could not extract YDB_SDK_VERSION from src/version.h" >&2 - exit 1 - fi - SDK_DEB_VERSION="${SDK_VERSION}-1~${SERIES}1" - SDK_PKG="ydb-cpp-sdk" - - WORK_DIR_SDK=$(mktemp -d) - trap "rm -rf $WORK_DIR_SDK" EXIT - - SDK_BUILD_DIR="$WORK_DIR_SDK/${SDK_PKG}-${SDK_VERSION}" - - # Create source tarball from git archive (includes submodules content) - echo "Creating source tarball..." - cd "$REPO_ROOT" - git archive --format=tar --prefix="${SDK_PKG}-${SDK_VERSION}/" HEAD \ - | tar -xf - -C "$WORK_DIR_SDK" - - # Copy submodule contents (git archive doesn't include them) - if [[ -d "$REPO_ROOT/third_party/api-common-protos/google" ]]; then - mkdir -p "$SDK_BUILD_DIR/third_party/api-common-protos" - cp -r "$REPO_ROOT/third_party/api-common-protos/google" \ - "$SDK_BUILD_DIR/third_party/api-common-protos/" - fi - if [[ -d "$REPO_ROOT/third_party/FastLZ" ]]; then - mkdir -p "$SDK_BUILD_DIR/third_party/FastLZ" - cp -r "$REPO_ROOT/third_party/FastLZ/"* "$SDK_BUILD_DIR/third_party/FastLZ/" 2>/dev/null || true - fi - - # Generate debian/ directory - bash "$SCRIPT_DIR/generate-debian-directory.sh" --series "$SERIES" - cp -r "$REPO_ROOT/debian" "$SDK_BUILD_DIR/" - - # Create .orig.tar.gz - ORIG_TAR_SDK="$WORK_DIR_SDK/${SDK_PKG}_${SDK_VERSION}.orig.tar.gz" - tar -czf "$ORIG_TAR_SDK" -C "$WORK_DIR_SDK" "${SDK_PKG}-${SDK_VERSION}" - - # Build source package - cd "$SDK_BUILD_DIR" - debuild -S "$DEBUILD_ORIG_FLAG" "${DEBUILD_SIGN_ARGS[@]}" - - # Upload - CHANGES_FILE_SDK="$WORK_DIR_SDK/${SDK_PKG}_${SDK_DEB_VERSION}_source.changes" - if [[ -f "$CHANGES_FILE_SDK" ]]; then - do_upload "$CHANGES_FILE_SDK" - else - CHANGES_FILE_SDK=$(ls "$WORK_DIR_SDK"/${SDK_PKG}_*_source.changes 2>/dev/null | head -1) - if [[ -n "$CHANGES_FILE_SDK" ]]; then - do_upload "$CHANGES_FILE_SDK" - else - echo "Error: no _source.changes file found for ydb-cpp-sdk." >&2 - exit 1 - fi - fi - - cd "$REPO_ROOT" - echo "ydb-cpp-sdk source package built successfully." - echo "" -fi - -echo "============================================================" -echo "Done." -if $DRY_RUN; then - echo "(dry-run mode — nothing was uploaded)" -fi -echo "============================================================" From 96b812c2268855bb39df5232aa4153c059afc695 Mon Sep 17 00:00:00 2001 From: Artem Ermoshkin Date: Thu, 21 May 2026 15:17:56 +0300 Subject: [PATCH 13/13] fix build errors --- scripts/generate-debian-directory.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/generate-debian-directory.sh b/scripts/generate-debian-directory.sh index 0a2f2c9a79d..414140e6b4e 100755 --- a/scripts/generate-debian-directory.sh +++ b/scripts/generate-debian-directory.sh @@ -82,8 +82,7 @@ Build-Depends: debhelper-compat (= 13), libre2-dev, libc-ares-dev, rapidjson-dev, - yandex-googleapis-api-common-protos, - yandex-opentelemetry-cpp-dev + yandex-googleapis-api-common-protos Standards-Version: 4.6.2 Homepage: https://ydb.tech Rules-Requires-Root: no @@ -119,13 +118,13 @@ Description: YDB C++ SDK IAM plugin development files Package: libydb-cpp-otel-metrics-dev Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}), yandex-opentelemetry-cpp-dev +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) Description: YDB C++ SDK OpenTelemetry metrics plugin development files Static library and headers for YDB C++ SDK OpenTelemetry metrics plugin. Package: libydb-cpp-otel-tracing-dev Architecture: any -Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}), yandex-opentelemetry-cpp-dev +Depends: \${misc:Depends}, libydb-cpp-dev (= \${binary:Version}) Description: YDB C++ SDK OpenTelemetry tracing plugin development files Static library and headers for YDB C++ SDK OpenTelemetry tracing plugin. EOF_CONTROL @@ -164,6 +163,10 @@ debian/tmp/usr/share/yandex/lib/*/cmake/base64 usr/share/yandex/lib/*/cmake/ debian/tmp/usr/share/yandex/include/picojson usr/share/yandex/include/ debian/tmp/usr/share/yandex/include/jwt-cpp usr/share/yandex/include/ debian/tmp/usr/share/yandex/cmake/jwt-cpp* usr/share/yandex/cmake/ +debian/tmp/usr/share/yandex/include/opentelemetry usr/share/yandex/include/ +debian/tmp/usr/share/yandex/lib/*/libopentelemetry_* usr/share/yandex/lib/ +debian/tmp/usr/share/yandex/lib/*/cmake/opentelemetry-cpp usr/share/yandex/lib/*/cmake/ +debian/tmp/usr/share/yandex/lib/*/pkgconfig/opentelemetry_*.pc usr/share/yandex/lib/*/pkgconfig/ EOF_INSTALL cat < debian/libydb-cpp-iam-dev.install