Create .deb packages for ydb c++ sdk and plugins.

.github/actions/prepare_vm/action.yaml

@@ -67,15 +67,6 @@ runs:
         cmake --install . --config Release --prefix ~/ydb_deps/grpc
         cd ../../
 
-        # Install base64
-        wget -O base64-0.5.2.tar.gz https://github.com/aklomp/base64/archive/refs/tags/v0.5.2.tar.gz
-        tar -xvzf base64-0.5.2.tar.gz && cd base64-0.5.2
-        mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release ..
-        cmake --build . --config Release
-        cmake --install . --config Release --prefix ~/ydb_deps/base64
-        cd ../../
-
         # Install brotli
         wget -O brotli-1.1.0.tar.gz https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz
         tar -xvzf brotli-1.1.0.tar.gz && cd brotli-1.1.0
@@ -85,17 +76,8 @@ runs:
         cmake --install . --config Release --prefix ~/ydb_deps/brotli
         cd ../../
 
-        # Install jwt-cpp
-        wget -O jwt-cpp-0.7.0.tar.gz https://github.com/Thalhammer/jwt-cpp/archive/refs/tags/v0.7.0.tar.gz
-        tar -xvzf jwt-cpp-0.7.0.tar.gz && cd jwt-cpp-0.7.0
-        mkdir build && cd build
-        cmake -G Ninja ${ENABLE_CCACHE} -DCMAKE_BUILD_TYPE=Release ..
-        cmake --build . --config Release
-        cmake --install . --config Release --prefix ~/ydb_deps/jwt-cpp
-        cd ../../
-
         # Clean up
         ccache -s
         sudo rm -rf llvm.sh abseil-cpp-20230802.0.tar.gz protobuf-3.21.12.tar.gz grpc-1.54.3.tar.gz \
-          base64-0.5.2.tar.gz brotli-1.1.0.tar.gz jwt-cpp-0.7.0.tar.gz abseil-cpp-20230802.0 \
-          protobuf-3.21.12 grpc-1.54.3 base64-0.5.2 brotli-1.1.0 jwt-cpp-0.7.0
+          brotli-1.1.0.tar.gz abseil-cpp-20230802.0 \
+          protobuf-3.21.12 grpc-1.54.3 brotli-1.1.0

.github/workflows/release_publish.yaml

@@ -0,0 +1,128 @@
+name: Publish release artifacts
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Release tag to attach assets to (e.g. v1.2.3)"
+        required: true
+        type: string
+
+concurrency:
+  group: release-publish-${{ github.event.release.tag_name || inputs.tag }}
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  build-and-upload-debs:
+    name: "Build .deb packages and attach to release (Ubuntu 24.04 / noble)"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Resolve release tag
+        id: tag
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
+            echo "ref=${{ inputs.tag }}"  >> "$GITHUB_OUTPUT"
+          else
+            echo "tag=${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT"
+            echo "ref=${{ github.event.release.tag_name }}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+          ref: ${{ steps.tag.outputs.ref }}
+
+      - name: Build .deb packages in Ubuntu 24.04 container
+        shell: bash
+        run: |
+          mkdir -p artifacts
+          docker run --rm --network host \
+            -v "$PWD:/source" \
+            ubuntu:24.04 \
+            bash -c '
+              set -e
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update
+              apt-get install -y \
+                build-essential \
+                cmake \
+                pkg-config \
+                git \
+                libidn11-dev \
+                libssl-dev \
+                zlib1g-dev \
+                libprotobuf-dev \
+                protobuf-compiler \
+                libgrpc++-dev \
+                protobuf-compiler-grpc \
+                libbrotli-dev \
+                liblz4-dev \
+                libzstd-dev \
+                libbz2-dev \
+                libxxhash-dev \
+                libsnappy-dev \
+                libdouble-conversion-dev \
+                libgtest-dev \
+                libre2-dev \
+                libc-ares-dev \
+                rapidjson-dev \
+                python3 \
+                python3-six \
+                ragel \
+                yasm
+
+              cd /source
+              cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex
+              cmake --build build_googleapis_deb -j$(nproc)
+              cmake --build build_googleapis_deb --target package
+              dpkg -i build_googleapis_deb/*.deb
+
+              ./scripts/generate-debian-directory.sh
+              cmake -S . -B build-deb \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DYDB_SDK_INSTALL=ON \
+                -DYDB_SDK_EXAMPLES=OFF \
+                -DYDB_SDK_TESTS=OFF \
+                -DYDB_SDK_ENABLE_OTEL_METRICS=ON \
+                -DYDB_SDK_ENABLE_OTEL_TRACE=ON \
+                -DBUILD_SHARED_LIBS=OFF \
+                -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
+                -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
+                -DCMAKE_PREFIX_PATH="/usr/share/yandex"
+              cmake --build build-deb --target package -j$(nproc)
+
+              cp build_googleapis_deb/*.deb /source/artifacts/
+              cp build-deb/*.deb            /source/artifacts/
+            '
+
+      - name: Smoke-test generated .deb packages
+        shell: bash
+        run: |
+          ./scripts/test_deb_packages.sh artifacts
+
+      - name: List built artifacts
+        shell: bash
+        run: ls -la artifacts/
+
+      - name: Upload .deb files to GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        shell: bash
+        run: |
+          gh release upload "${{ steps.tag.outputs.tag }}" artifacts/*.deb --clobber
+
+      - name: Upload .deb files as workflow artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: deb-packages-${{ steps.tag.outputs.tag }}
+          path: artifacts/*.deb
+          if-no-files-found: warn
+          retention-days: 30

.github/workflows/tests.yaml

@@ -177,4 +177,94 @@ jobs:
           if [ -z "$(ls -A ./_install)" ]; then
             echo "Error: Installation directory is empty!"
             exit 1
-          fi
+          fi
+
+  deb-packages:
+    name: "Test Debian Packages"
+    concurrency:
+      group: deb-packages-${{ github.ref }}
+      cancel-in-progress: true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout PR
+        uses: actions/checkout@v4
+        if: github.event.pull_request.head.sha != ''
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        if: github.event.pull_request.head.sha == ''
+        with:
+          submodules: true
+
+      - name: Validate dpkg-buildpackage
+        shell: bash
+        run: |
+          ./scripts/test_dpkg_buildpackage.sh
+
+      - name: Build CPack .deb packages
+        shell: bash
+        run: |
+          docker run --rm --network host \
+            -v "$PWD:/source" \
+            ubuntu:24.04 \
+            bash -c '
+              set -e
+              export DEBIAN_FRONTEND=noninteractive
+              apt-get update
+              apt-get install -y \
+                build-essential \
+                cmake \
+                pkg-config \
+                git \
+                libidn11-dev \
+                libssl-dev \
+                zlib1g-dev \
+                libprotobuf-dev \
+                protobuf-compiler \
+                libgrpc++-dev \
+                protobuf-compiler-grpc \
+                libbrotli-dev \
+                liblz4-dev \
+                libzstd-dev \
+                libbz2-dev \
+                libxxhash-dev \
+                libsnappy-dev \
+                libdouble-conversion-dev \
+                libgtest-dev \
+                libre2-dev \
+                libc-ares-dev \
+                rapidjson-dev \
+                python3 \
+                python3-six \
+                ragel \
+                yasm
+
+              cd /source
+              cmake -S scripts/googleapis_deb -B build_googleapis_deb -DCMAKE_INSTALL_PREFIX=/usr/share/yandex
+              cmake --build build_googleapis_deb -j$(nproc)
+              cmake --build build_googleapis_deb --target package
+              dpkg -i build_googleapis_deb/*.deb
+
+              ./scripts/generate-debian-directory.sh
+              cmake -S . -B build-deb \
+                -DCMAKE_BUILD_TYPE=Release \
+                -DYDB_SDK_INSTALL=ON \
+                -DYDB_SDK_EXAMPLES=OFF \
+                -DYDB_SDK_TESTS=OFF \
+                -DYDB_SDK_ENABLE_OTEL_METRICS=ON \
+                -DYDB_SDK_ENABLE_OTEL_TRACE=ON \
+                -DBUILD_SHARED_LIBS=OFF \
+                -DYDB_SDK_USE_SYSTEM_GOOGLEAPIS=ON \
+                -DCMAKE_INSTALL_PREFIX=/usr/share/yandex \
+                -DCMAKE_PREFIX_PATH="/usr/share/yandex"
+              cmake --build build-deb --target package -j$(nproc)
+              cp build_googleapis_deb/*.deb build-deb/
+            '
+
+      - name: Smoke-test generated .deb packages
+        shell: bash
+        run: |
+          ./scripts/test_deb_packages.sh build-deb

.gitignore

@@ -83,3 +83,4 @@ CMakeUserPresets.json
 
 # Dev Containers
 !Dockerfile
+debian/

.gitmodules

@@ -4,3 +4,12 @@
 [submodule "third_party/FastLZ"]
 	path = third_party/FastLZ
 	url = https://github.com/ariya/FastLZ
+[submodule "third_party/aklomp-base64"]
+	path = third_party/aklomp-base64
+	url = https://github.com/aklomp/base64.git
+[submodule "third_party/jwt-cpp"]
+	path = third_party/jwt-cpp
+	url = https://github.com/Thalhammer/jwt-cpp.git
+[submodule "third_party/opentelemetry-cpp"]
+	path = third_party/opentelemetry-cpp
+	url = https://github.com/open-telemetry/opentelemetry-cpp.git

CMakeLists.txt

@@ -73,12 +73,83 @@ if (YDB_SDK_TESTS)
 endif()
 
 if (YDB_SDK_INSTALL)
+  function(_ydb_sdk_create_package_library PackageProp TargetName ComponentName)
+    foreach(Tgt IN LISTS YDB_CPP_${PackageProp}_COMPONENT_TARGETS)
+      _ydb_sdk_collect_package_target(${PackageProp} ${Tgt})
+    endforeach()
+
+    get_property(PackageSources GLOBAL PROPERTY YDB_CPP_${PackageProp}_SOURCES)
+    if (NOT PackageSources)
+      return()
+    endif()
+
+    get_property(PackageInternalDeps GLOBAL PROPERTY YDB_CPP_${PackageProp}_INTERNAL_DEPS)
+    get_property(PackagePublicDeps GLOBAL PROPERTY YDB_CPP_${PackageProp}_PUBLIC_DEPS)
+    get_property(PackagePublicOpts GLOBAL PROPERTY YDB_CPP_${PackageProp}_PUBLIC_OPTS)
+    get_property(PackagePublicDefs GLOBAL PROPERTY YDB_CPP_${PackageProp}_PUBLIC_DEFS)
+    get_property(PackageIncludeDirs GLOBAL PROPERTY YDB_CPP_${PackageProp}_INCLUDE_DIRS)
+
+    list(REMOVE_DUPLICATES PackageSources)
+    if (PackageInternalDeps)
+      list(REMOVE_DUPLICATES PackageInternalDeps)
+    endif()
+    if (PackagePublicDeps)
+      list(REMOVE_DUPLICATES PackagePublicDeps)
+      list(REMOVE_ITEM PackagePublicDeps ${PackageInternalDeps})
+    endif()
+    if (PackagePublicOpts)
+      list(REMOVE_DUPLICATES PackagePublicOpts)
+    endif()
+    if (PackagePublicDefs)
+      list(REMOVE_DUPLICATES PackagePublicDefs)
+    endif()
+    if (PackageIncludeDirs)
+      list(REMOVE_DUPLICATES PackageIncludeDirs)
+    endif()
+
+    foreach(src IN LISTS PackageSources)
+      if (NOT EXISTS "${src}")
+        set_source_files_properties("${src}" PROPERTIES GENERATED TRUE)
+      endif()
+    endforeach()
+
+    add_library(${TargetName} STATIC ${PackageSources})
+    target_include_directories(${TargetName} PUBLIC
+      $<BUILD_INTERFACE:${YDB_SDK_SOURCE_DIR}>
+      $<BUILD_INTERFACE:${YDB_SDK_BINARY_DIR}>
+      $<BUILD_INTERFACE:${YDB_SDK_SOURCE_DIR}/include>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
+      $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/__ydb_sdk_special_headers>
+    )
+    target_include_directories(${TargetName} PRIVATE
+      ${PackageIncludeDirs}
+    )
+    if (TargetName STREQUAL "libydb-cpp")
+      set_target_properties(${TargetName} PROPERTIES OUTPUT_NAME ydb-cpp)
+    endif()
+    target_compile_definitions(${TargetName} PUBLIC YDB_SDK_OSS ${PackagePublicDefs})
+    target_link_libraries(${TargetName} PUBLIC ${PackagePublicDeps})
+    if (PackageInternalDeps)
+      add_dependencies(${TargetName} ${PackageInternalDeps})
+    endif()
+    if (PackagePublicOpts)
+      target_compile_options(${TargetName} PUBLIC ${PackagePublicOpts})
+    endif()
+    _ydb_sdk_install_targets(TARGETS ${TargetName} COMPONENT ${ComponentName})
+  endfunction()
+
+  _ydb_sdk_create_package_library(CORE libydb-cpp libydb-cpp)
+  _ydb_sdk_create_package_library(LIBYDB_CPP_IAM ydb-cpp-iam libydb-cpp-iam)
+  _ydb_sdk_create_package_library(LIBYDB_CPP_OTEL_METRICS ydb-cpp-otel-metrics libydb-cpp-otel-metrics)
+  _ydb_sdk_create_package_library(LIBYDB_CPP_OTEL_TRACING ydb-cpp-otel-tracing libydb-cpp-otel-tracing)
+
   _ydb_sdk_install_headers(${CMAKE_INSTALL_INCLUDEDIR})
   install(EXPORT ydb-cpp-sdk-targets
     FILE ydb-cpp-sdk-targets.cmake
     CONFIGURATIONS RELEASE
     NAMESPACE YDB-CPP-SDK::
     DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/ydb-cpp-sdk/release
+    COMPONENT libydb-cpp
   )
   configure_package_config_file(
     ${YDB_SDK_SOURCE_DIR}/cmake/ydb-cpp-sdk-config.cmake.in

CMakePresets.json

@@ -11,7 +11,7 @@
       "displayName": "Generic Config",
       "cacheVariables": {
         "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
-        "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/base64;~/ydb_deps/brotli;~/ydb_deps/jwt-cpp"
+        "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/brotli"
       },
       "generator": "Ninja",
       "binaryDir": "${sourceDir}/build"
@@ -65,6 +65,22 @@
       "inherits": ["release-base", "clang-toolchain"],
       "displayName": "Default Release Config (Clang)"
     },
+    {
+      "name": "package-deb-clang",
+      "inherits": ["release-base", "clang-toolchain"],
+      "displayName": "Package Config (Clang) for .deb",
+      "binaryDir": "${sourceDir}/build-deb",
+      "cacheVariables": {
+        "YDB_SDK_INSTALL": "ON",
+        "YDB_SDK_ENABLE_OTEL_METRICS": "ON",
+        "YDB_SDK_ENABLE_OTEL_TRACE": "ON",
+        "YDB_SDK_EXAMPLES": "OFF",
+        "BUILD_SHARED_LIBS": "OFF",
+        "CMAKE_EXE_LINKER_FLAGS": "",
+        "CMAKE_SHARED_LINKER_FLAGS": "",
+        "CMAKE_PREFIX_PATH": "~/ydb_deps/absl;~/ydb_deps/protobuf;~/ydb_deps/grpc;~/ydb_deps/brotli;~/ydb_deps/opentelemetry-cpp-install"
+      }
+    },
     {
       "name": "release-gcc",
       "inherits": ["release-base", "gcc-toolchain"],