diff --git a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml b/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml deleted file mode 100644 index 041eabc252b..00000000000 --- a/.github/workflows/build-deb-cloudberry-ubuntu24.04.yml +++ /dev/null @@ -1,1892 +0,0 @@ -# -------------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed -# with this work for additional information regarding copyright -# ownership. The ASF licenses this file to You under the Apache -# License, Version 2.0 (the "License"); you may not use this file -# except in compliance with the License. You may obtain a copy of the -# License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. See the License for the specific language governing -# permissions and limitations under the License. -# -# -------------------------------------------------------------------- -# GitHub Actions Workflow: Apache Cloudberry Build Pipeline -# -------------------------------------------------------------------- -# Description: -# -# This workflow builds, tests, and packages Apache Cloudberry on -# Ubuntu 24.04. It ensures artifact integrity and performs installation -# tests. -# -# Workflow Overview: -# 1. **Build Job**: -# - Configures and builds Apache Cloudberry. -# - Supports debug build configuration via ENABLE_DEBUG flag. -# - Runs unit tests and verifies build artifacts. -# - Creates DEB packages (regular and debug), source tarball -# and additional files for dupload utility. -# - **Key Artifacts**: DEB package, source tarball, changes and dsc files, build logs. -# -# 2. **DEB Install Test Job**: -# - Verifies DEB integrity and installs Cloudberry. -# - Validates successful installation. -# - **Key Artifacts**: Installation logs, verification results. -# -# 3. **Report Job**: -# - Aggregates job results into a final report. -# - Sends failure notifications if any step fails. -# -# Execution Environment: -# - **Runs On**: ubuntu-22.04 with ubuntu-24.04 containers. -# - **Resource Requirements**: -# - Disk: Minimum 20GB free space. -# - Memory: Minimum 8GB RAM. -# - CPU: Recommended 4+ cores. -# -# Triggers: -# - Push to `main` branch. -# - Pull request that modifies this workflow file. -# - Scheduled: Every Monday at 02:00 UTC. -# - Manual workflow dispatch. -# -# Container Images: -# - **Build**: `apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest` -# - **Test**: `apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest` -# -# Artifacts: -# - DEB Package (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Changes and DSC files (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Source Tarball (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# - Logs and Test Results (retention: ${{ env.LOG_RETENTION_DAYS }} days). -# -# Notes: -# - Supports concurrent job execution. -# - Supports debug builds with preserved symbols. -# -------------------------------------------------------------------- - -name: Apache Cloudberry Debian Build - -on: - push: - branches: [main, REL_2_STABLE] - pull_request: - paths: - - '.github/workflows/build-deb-cloudberry-ubuntu24.04.yml' - # We can enable the PR test when needed - # branches: [main, REL_2_STABLE] - # types: [opened, synchronize, reopened, edited] - schedule: - # Run every Monday at 02:00 UTC - - cron: '0 2 * * 1' - workflow_dispatch: # Manual trigger - inputs: - test_selection: - description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' - required: false - default: 'all' - type: string - reuse_artifacts_from_run_id: - description: 'Reuse build artifacts from a previous run ID (leave empty to build fresh)' - required: false - default: '' - type: string - -# Note: Step details, logs, and artifacts require users to be logged into GitHub -# even for public repositories. This is a GitHub security feature and cannot -# be overridden by permissions. - -permissions: - # READ permissions allow viewing repository contents - contents: read # Required for checking out code and reading repository files - - # READ permissions for packages (Container registry, etc) - packages: read # Allows reading from GitHub package registry - - # WRITE permissions for actions includes read access to: - # - Workflow runs - # - Artifacts (requires GitHub login) - # - Logs (requires GitHub login) - actions: write - - # READ permissions for checks API: - # - Step details visibility (requires GitHub login) - # - Check run status and details - checks: read - - # READ permissions for pull request metadata: - # - PR status - # - Associated checks - # - Review states - pull-requests: read - -env: - LOG_RETENTION_DAYS: 7 - ENABLE_DEBUG: false - -jobs: - - ## ====================================================================== - ## Job: check-skip - ## ====================================================================== - - check-skip: - runs-on: ubuntu-22.04 - outputs: - should_skip: ${{ steps.skip-check.outputs.should_skip }} - steps: - - id: skip-check - shell: bash - env: - EVENT_NAME: ${{ github.event_name }} - PR_TITLE: ${{ github.event.pull_request.title || '' }} - PR_BODY: ${{ github.event.pull_request.body || '' }} - run: | - # Default to not skipping - echo "should_skip=false" >> "$GITHUB_OUTPUT" - - # Apply skip logic only for pull_request events - if [[ "$EVENT_NAME" == "pull_request" ]]; then - # Combine PR title and body for skip check - MESSAGE="${PR_TITLE}\n${PR_BODY}" - - # Escape special characters using printf %s - ESCAPED_MESSAGE=$(printf "%s" "$MESSAGE") - - echo "Checking PR title and body (escaped): $ESCAPED_MESSAGE" - - # Check for skip patterns - if echo -e "$ESCAPED_MESSAGE" | grep -qEi '\[skip[ -]ci\]|\[ci[ -]skip\]|\[no[ -]ci\]'; then - echo "should_skip=true" >> "$GITHUB_OUTPUT" - fi - else - echo "Skip logic is not applied for $EVENT_NAME events." - fi - - - name: Report Skip Status - if: steps.skip-check.outputs.should_skip == 'true' - run: | - echo "CI Skip flag detected in PR - skipping all checks." - exit 0 - - ## ====================================================================== - ## Job: prepare-test-matrix-deb - ## ====================================================================== - - prepare-test-matrix-deb: - runs-on: ubuntu-22.04 - needs: [check-skip] - if: needs.check-skip.outputs.should_skip != 'true' - outputs: - test-matrix: ${{ steps.set-matrix.outputs.matrix }} - - steps: - - id: set-matrix - run: | - echo "=== Matrix Preparation Diagnostics ===" - echo "Event type: ${{ github.event_name }}" - echo "Test selection input: '${{ github.event.inputs.test_selection }}'" - - # Define defaults - DEFAULT_NUM_PRIMARY_MIRROR_PAIRS=3 - DEFAULT_ENABLE_CGROUPS=false - DEFAULT_ENABLE_CORE_CHECK=true - DEFAULT_PG_SETTINGS_OPTIMIZER="" - - # Define base test configurations - ALL_TESTS='{ - "include": [ - {"test":"ic-deb-good-opt-off", - "make_configs":["src/test/regress:installcheck-good"], - "pg_settings":{"optimizer":"off"} - }, - {"test":"ic-deb-good-opt-on", - "make_configs":["src/test/regress:installcheck-good"], - "pg_settings":{"optimizer":"on"} - }, - {"test":"pax-ic-deb-good-opt-off", - "make_configs":[ - "contrib/pax_storage/:pax-test", - "contrib/pax_storage/:regress_test" - ], - "pg_settings":{ - "optimizer":"off", - "default_table_access_method":"pax" - } - }, - {"test":"pax-ic-deb-good-opt-on", - "make_configs":[ - "contrib/pax_storage/:pax-test", - "contrib/pax_storage/:regress_test" - ], - "pg_settings":{ - "optimizer":"on", - "default_table_access_method":"pax" - } - }, - {"test":"ic-deb-contrib", - "make_configs":["contrib/auto_explain:installcheck", - "contrib/amcheck:installcheck", - "contrib/citext:installcheck", - "contrib/btree_gin:installcheck", - "contrib/btree_gist:installcheck", - "contrib/dblink:installcheck", - "contrib/dict_int:installcheck", - "contrib/dict_xsyn:installcheck", - "contrib/extprotocol:installcheck", - "contrib/file_fdw:installcheck", - "contrib/formatter_fixedwidth:installcheck", - "contrib/hstore:installcheck", - "contrib/indexscan:installcheck", - "contrib/pg_trgm:installcheck", - "contrib/indexscan:installcheck", - "contrib/pgcrypto:installcheck", - "contrib/pgstattuple:installcheck", - "contrib/tablefunc:installcheck", - "contrib/passwordcheck:installcheck", - "contrib/pg_buffercache:installcheck", - "contrib/sslinfo:installcheck"] - }, - {"test":"ic-deb-gpcontrib", - "make_configs":["gpcontrib/orafce:installcheck", - "gpcontrib/zstd:installcheck", - "gpcontrib/gp_sparse_vector:installcheck", - "gpcontrib/gp_toolkit:installcheck"] - }, - {"test":"gpcontrib-gp-stats-collector", - "make_configs":["gpcontrib/gp_stats_collector:installcheck"], - "extension":"gp_stats_collector" - }, - {"test":"ic-cbdb-parallel", - "make_configs":["src/test/regress:installcheck-cbdb-parallel"] - } - ] - }' - - # Function to apply defaults - apply_defaults() { - echo "$1" | jq --arg npm "$DEFAULT_NUM_PRIMARY_MIRROR_PAIRS" \ - --argjson ec "$DEFAULT_ENABLE_CGROUPS" \ - --argjson ecc "$DEFAULT_ENABLE_CORE_CHECK" \ - --arg opt "$DEFAULT_PG_SETTINGS_OPTIMIZER" \ - 'def get_defaults: - { - num_primary_mirror_pairs: ($npm|tonumber), - enable_cgroups: $ec, - enable_core_check: $ecc, - pg_settings: { - optimizer: $opt - } - }; - get_defaults * .' - } - - # Extract all valid test names from ALL_TESTS - VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') - - # Parse input test selection - IFS=',' read -ra SELECTED_TESTS <<< "${{ github.event.inputs.test_selection }}" - - # Default to all tests if selection is empty or 'all' - if [[ "${SELECTED_TESTS[*]}" == "all" || -z "${SELECTED_TESTS[*]}" ]]; then - mapfile -t SELECTED_TESTS <<< "$VALID_TESTS" - fi - - # Validate and filter selected tests - INVALID_TESTS=() - FILTERED_TESTS=() - for TEST in "${SELECTED_TESTS[@]}"; do - TEST=$(echo "$TEST" | tr -d '[:space:]') # Trim whitespace - if echo "$VALID_TESTS" | grep -qw "$TEST"; then - FILTERED_TESTS+=("$TEST") - else - INVALID_TESTS+=("$TEST") - fi - done - - # Handle invalid tests - if [[ ${#INVALID_TESTS[@]} -gt 0 ]]; then - echo "::error::Invalid test(s) selected: ${INVALID_TESTS[*]}" - echo "Valid tests are: $(echo "$VALID_TESTS" | tr '\n' ', ')" - exit 1 - fi - - # Build result JSON with defaults applied - RESULT='{"include":[' - FIRST=true - for TEST in "${FILTERED_TESTS[@]}"; do - CONFIG=$(jq -c --arg test "$TEST" '.include[] | select(.test == $test)' <<< "$ALL_TESTS") - FILTERED_WITH_DEFAULTS=$(apply_defaults "$CONFIG") - if [[ "$FIRST" == true ]]; then - FIRST=false - else - RESULT="${RESULT}," - fi - RESULT="${RESULT}${FILTERED_WITH_DEFAULTS}" - done - RESULT="${RESULT}]}" - - # Output the matrix for GitHub Actions - echo "Final matrix configuration:" - echo "$RESULT" | jq . - - # Fix: Use block redirection - { - echo "matrix<> "$GITHUB_OUTPUT" - - echo "=== Matrix Preparation Complete ===" - - ## ====================================================================== - ## Job: build-deb - ## ====================================================================== - - build-deb: - name: Build Apache Cloudberry DEB (Ubuntu 24.04) - env: - JOB_TYPE: build - needs: [check-skip] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - if: github.event.inputs.reuse_artifacts_from_run_id == '' - outputs: - build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} - - container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest - options: >- - --user root - -h cdw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "Build skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Set build timestamp - id: set_timestamp # Add an ID to reference this step - run: | - timestamp=$(date +'%Y%m%d_%H%M%S') - echo "timestamp=$timestamp" | tee -a "$GITHUB_OUTPUT" # Use GITHUB_OUTPUT for job outputs - echo "BUILD_TIMESTAMP=$timestamp" | tee -a "$GITHUB_ENV" # Also set as environment variable - - - name: Checkout Apache Cloudberry - uses: actions/checkout@v4 - with: - fetch-depth: 1 - submodules: true - - - name: Cloudberry Environment Initialization - shell: bash - env: - LOGS_DIR: build-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Generate Build Job Summary Start - run: | - { - echo "# Build Job Summary (Ubuntu 24.04)" - echo "## Environment" - echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - echo "- ENABLE_DEBUG: ${{ env.ENABLE_DEBUG }}" - echo "- OS Version: $(lsb_release -sd)" - echo "- GCC Version: $(gcc --version | head -n1)" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Run Apache Cloudberry configure script - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ENABLE_DEBUG=${{ env.ENABLE_DEBUG }} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh"; then - echo "::error::Configure script failed" - exit 1 - fi - - - name: Run Apache Cloudberry build script - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/build-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} BUILD_DESTINATION=${BUILD_DESTINATION} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/build-cloudberry.sh"; then - echo "::error::Build script failed" - exit 1 - fi - - - name: Verify build artifacts - shell: bash - run: | - set -eo pipefail - - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - echo "Verifying build artifacts..." - { - echo "=== Build Artifacts Verification ===" - echo "Timestamp: $(date -u)" - - if [ ! -d "${BUILD_DESTINATION}" ]; then - echo "::error::Build artifacts directory not found" - exit 1 - fi - - # Verify critical binaries - critical_binaries=( - "${BUILD_DESTINATION}/bin/postgres" - "${BUILD_DESTINATION}/bin/psql" - ) - - echo "Checking critical binaries..." - for binary in "${critical_binaries[@]}"; do - if [ ! -f "$binary" ]; then - echo "::error::Critical binary missing: $binary" - exit 1 - fi - if [ ! -x "$binary" ]; then - echo "::error::Binary not executable: $binary" - exit 1 - fi - echo "Binary verified: $binary" - ls -l "$binary" - done - - # Test binary execution - echo "Testing binary execution..." - if ! ${BUILD_DESTINATION}/bin/postgres --version; then - echo "::error::postgres binary verification failed" - exit 1 - fi - if ! ${BUILD_DESTINATION}/bin/psql --version; then - echo "::error::psql binary verification failed" - exit 1 - fi - - echo "All build artifacts verified successfully" - } 2>&1 | tee -a build-logs/details/build-verification.log - - - name: Create Source tarball, create DEB and verify artifacts - shell: bash - env: - CBDB_VERSION: 99.0.0 - BUILD_NUMBER: 1 - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - echo "=== Artifact Creation Log ===" - echo "Timestamp: $(date -u)" - - cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu24.04/* debian/ - chown -R "$(whoami)" debian - chmod -x debian/*install - - # replace not supported symbols in version - CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/\//./g") - CBDB_VERSION=$(echo "$CBDB_VERSION" | sed "s/_/-/g") - - echo "We will built ${CBDB_VERSION}" - export BUILD_DESTINATION=${SRC_DIR}/debian/build - - if ! ${SRC_DIR}/devops/build/packaging/deb/build-deb.sh -v $CBDB_VERSION; then - echo "::error::Build script failed" - exit 1 - fi - - ARCH=$(dpkg --print-architecture) - # Detect OS distribution (e.g., ubuntu24.04, debian12) - if [ -f /etc/os-release ]; then - . /etc/os-release - OS_DISTRO=$(echo "${ID}${VERSION_ID}" | tr '[:upper:]' '[:lower:]') - else - OS_DISTRO="unknown" - fi - CBDB_PKG_VERSION=${CBDB_VERSION}-${BUILD_NUMBER}-${OS_DISTRO} - - echo "Produced artifacts" - ls -l ../ - - echo "Copy artifacts to subdirectory for sign/upload" - mkdir ${SRC_DIR}/deb - DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".deb - DBG_DEB_FILE="apache-cloudberry-db-incubating-dbgsym_${CBDB_PKG_VERSION}"_"${ARCH}".ddeb - CHANGES_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".changes - BUILDINFO_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}"_"${ARCH}".buildinfo - DSC_DEB_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".dsc - SOURCE_FILE="apache-cloudberry-db-incubating_${CBDB_PKG_VERSION}".tar.xz - cp ../"${DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${DBG_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${CHANGES_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${BUILDINFO_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${DSC_DEB_FILE}" "${SRC_DIR}/deb" - cp ../"${SOURCE_FILE}" "${SRC_DIR}/deb" - mkdir "${SRC_DIR}/deb/debian" - cp debian/changelog "${SRC_DIR}/deb/debian" - - # Get package information - echo "Package Information:" - dpkg --info "${SRC_DIR}/deb/${DEB_FILE}" - dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" - - # Verify critical files in DEB - echo "Verifying critical files in DEB..." - for binary in "bin/postgres" "bin/psql"; do - if ! dpkg --contents "${SRC_DIR}/deb/${DEB_FILE}" | grep -c "${binary}$"; then - echo "::error::Critical binary '${binary}' not found in DEB" - exit 1 - fi - done - - # Record checksums - echo "Calculating checksums..." - sha256sum "${SRC_DIR}/deb/${DEB_FILE}" | tee -a build-logs/details/checksums.log - - echo "Artifacts created and verified successfully" - - - } 2>&1 | tee -a build-logs/details/artifact-creation.log - - - name: Run Apache Cloudberry unittest script - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh - if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/unittest-cloudberry.sh"; then - echo "::error::Unittest script failed" - exit 1 - fi - - - name: Generate Build Job Summary End - run: | - { - echo "## Build Results" - echo "- End Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Upload build logs - uses: actions/upload-artifact@v4 - with: - name: build-logs-ubuntu24.04-${{ env.BUILD_TIMESTAMP }} - path: | - build-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload Cloudberry DEB build artifacts - uses: actions/upload-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts - retention-days: ${{ env.LOG_RETENTION_DAYS }} - if-no-files-found: error - path: | - deb/*.deb - deb/*.ddeb - - - name: Upload Cloudberry deb source build artifacts - uses: actions/upload-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts - retention-days: ${{ env.LOG_RETENTION_DAYS }} - if-no-files-found: error - path: | - deb/*.tar.xz - deb/*.changes - deb/*.dsc - deb/*.buildinfo - deb/debian/changelog - - ## ====================================================================== - ## Job: deb-install-test - ## ====================================================================== - - deb-install-test: - name: DEB Install Test Apache Cloudberry (Ubuntu 24.04) - needs: [check-skip, build-deb] - if: | - !cancelled() && - (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') && - github.event.inputs.reuse_artifacts_from_run_id == '' - runs-on: ubuntu-22.04 - timeout-minutes: 120 - - container: - image: apache/incubator-cloudberry:cbdb-test-ubuntu24.04-latest - options: >- - --user root - -h cdw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "DEB install test skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Download Cloudberry DEB build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts - path: ${{ github.workspace }}/deb_build_artifacts - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - merge-multiple: false - - - name: Cloudberry Environment Initialization - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - LOGS_DIR: install-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Verify DEB artifacts - id: verify-artifacts - shell: bash - run: | - set -eo pipefail - - DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) - if [ ! -f "${DEB_FILE}" ]; then - echo "::error::DEB file not found" - exit 1 - fi - - echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying DEB artifacts..." - { - echo "=== DEB Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - - # Get DEB metadata and verify contents - echo "Package Information:" - dpkg-deb -f "${DEB_FILE}" - - # Get key DEB attributes for verification - DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) - DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) - echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" - echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" - - # Verify expected binaries are in the DEB - echo "Verifying critical files in DEB..." - for binary in "bin/postgres" "bin/psql"; do - if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then - echo "::error::Critical binary '${binary}' not found in DEB" - exit 1 - fi - done - - echo "DEB Details:" - echo "- Version: ${DEB_VERSION}" - echo "- Release: ${DEB_RELEASE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${DEB_FILE}" - - } 2>&1 | tee -a install-logs/details/deb-verification.log - - - name: Install Cloudberry DEB - shell: bash - env: - DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} - DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} - DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} - run: | - set -eo pipefail - - if [ -z "${DEB_FILE}" ]; then - echo "::error::DEB_FILE environment variable is not set" - exit 1 - fi - - { - echo "=== DEB Installation Log ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - echo "Version: ${DEB_VERSION}" - echo "Release: ${DEB_RELEASE}" - - # Clean install location - rm -rf /usr/local/cloudberry-db - - # Install DEB - echo "Starting installation..." - apt-get update - if ! apt-get -y install "${DEB_FILE}"; then - echo "::error::DEB installation failed" - exit 1 - fi - - # Change ownership back to gpadmin - it is needed for future tests - chown -R gpadmin:gpadmin /usr/local/cloudberry-db - - echo "Installation completed successfully" - dpkg-query -s apache-cloudberry-db-incubating - echo "Installed files:" - dpkg-query -L apache-cloudberry-db-incubating - } 2>&1 | tee -a install-logs/details/deb-installation.log - - - name: Upload install logs - uses: actions/upload-artifact@v4 - with: - name: install-logs-${{ matrix.name }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - install-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Generate Install Test Job Summary End - if: always() - shell: bash {0} - run: | - { - echo "# Installed Package Summary (Ubuntu 24.04)" - echo "\`\`\`" - - dpkg-query -s apache-cloudberry-db-incubating - echo "\`\`\`" - } >> "$GITHUB_STEP_SUMMARY" || true - - ## ====================================================================== - ## Job: test-deb - ## ====================================================================== - - test-deb: - name: ${{ matrix.test }} (Ubuntu 24.04) - needs: [check-skip, build-deb, prepare-test-matrix-deb] - if: | - !cancelled() && - (needs.build-deb.result == 'success' || needs.build-deb.result == 'skipped') - runs-on: ubuntu-22.04 - timeout-minutes: 120 - # actionlint-allow matrix[*].pg_settings - strategy: - fail-fast: false # Continue with other tests if one fails - matrix: ${{ fromJson(needs.prepare-test-matrix-deb.outputs.test-matrix) }} - - container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu24.04-latest - options: >- - --privileged - --user root - --hostname cdw - --shm-size=2gb - --ulimit core=-1 - --cgroupns=host - -v /sys/fs/cgroup:/sys/fs/cgroup:rw - -v /usr/share:/host_usr_share - -v /usr/local:/host_usr_local - -v /opt:/host_opt - - steps: - - name: Free Disk Space - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "=== Disk space before cleanup ===" - df -h / - - # Remove pre-installed tools from host to free disk space - rm -rf /host_opt/hostedtoolcache || true # GitHub Actions tool cache - rm -rf /host_usr_local/lib/android || true # Android SDK - rm -rf /host_usr_share/dotnet || true # .NET SDK - rm -rf /host_opt/ghc || true # Haskell GHC - rm -rf /host_usr_local/.ghcup || true # Haskell GHCup - rm -rf /host_usr_share/swift || true # Swift - rm -rf /host_usr_local/share/powershell || true # PowerShell - rm -rf /host_usr_local/share/chromium || true # Chromium - rm -rf /host_usr_share/miniconda || true # Miniconda - rm -rf /host_opt/az || true # Azure CLI - rm -rf /host_usr_share/sbt || true # Scala Build Tool - - echo "=== Disk space after cleanup ===" - df -h / - - - name: Skip Check - if: needs.check-skip.outputs.should_skip == 'true' - run: | - echo "Test ${{ matrix.test }} skipped via CI skip flag" >> "$GITHUB_STEP_SUMMARY" - exit 0 - - - name: Use timestamp from previous job - if: needs.check-skip.outputs.should_skip != 'true' - run: | - echo "Timestamp from output: ${{ needs.build-deb.outputs.build_timestamp }}" - - - name: Cloudberry Environment Initialization - shell: bash - env: - LOGS_DIR: build-logs - run: | - set -eo pipefail - if ! su - gpadmin -c "/tmp/init_system.sh"; then - echo "::error::Container initialization failed" - exit 1 - fi - - mkdir -p "${LOGS_DIR}/details" - chown -R gpadmin:gpadmin . - chmod -R 755 . - chmod 777 "${LOGS_DIR}" - - df -kh / - rm -rf /__t/* - df -kh / - - df -h | tee -a "${LOGS_DIR}/details/disk-usage.log" - free -h | tee -a "${LOGS_DIR}/details/memory-usage.log" - - { - echo "=== Environment Information ===" - uname -a - df -h - free -h - env - } | tee -a "${LOGS_DIR}/details/environment.log" - - echo "SRC_DIR=${GITHUB_WORKSPACE}" | tee -a "$GITHUB_ENV" - - - name: Setup cgroups - if: needs.check-skip.outputs.should_skip != 'true' - shell: bash - run: | - set -uxo pipefail - - if [ "${{ matrix.enable_cgroups }}" = "true" ]; then - - echo "Current mounts:" - mount | grep cgroup - - CGROUP_BASEDIR=/sys/fs/cgroup - - # 1. Basic setup with permissions - sudo chmod -R 777 ${CGROUP_BASEDIR}/ - sudo mkdir -p ${CGROUP_BASEDIR}/gpdb - sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb - sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb - - # 2. Enable controllers - sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/cgroup.subtree_control" || true - sudo bash -c "echo '+cpu +cpuset +memory +io' > ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control" || true - - # 3. CPU settings - sudo bash -c "echo 'max 100000' > ${CGROUP_BASEDIR}/gpdb/cpu.max" || true - sudo bash -c "echo '100' > ${CGROUP_BASEDIR}/gpdb/cpu.weight" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpu.weight.nice" || true - sudo bash -c "echo 0-$(( $(nproc) - 1 )) > ${CGROUP_BASEDIR}/gpdb/cpuset.cpus" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/cpuset.mems" || true - - # 4. Memory settings - sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.max" || true - sudo bash -c "echo '0' > ${CGROUP_BASEDIR}/gpdb/memory.min" || true - sudo bash -c "echo 'max' > ${CGROUP_BASEDIR}/gpdb/memory.high" || true - - # 5. IO settings - echo "Available block devices:" - lsblk - - sudo bash -c " - if [ -f \${CGROUP_BASEDIR}/gpdb/io.stat ]; then - echo 'Detected IO devices:' - cat \${CGROUP_BASEDIR}/gpdb/io.stat - fi - echo '' > \${CGROUP_BASEDIR}/gpdb/io.max || true - " - - # 6. Fix permissions again after all writes - sudo chmod -R 777 ${CGROUP_BASEDIR}/gpdb - sudo chown -R gpadmin:gpadmin ${CGROUP_BASEDIR}/gpdb - - # 7. Check required files - echo "Checking required files:" - required_files=( - "cgroup.procs" - "cpu.max" - "cpu.pressure" - "cpu.weight" - "cpu.weight.nice" - "cpu.stat" - "cpuset.cpus" - "cpuset.mems" - "cpuset.cpus.effective" - "cpuset.mems.effective" - "memory.current" - "io.max" - ) - - for file in "${required_files[@]}"; do - if [ -f "${CGROUP_BASEDIR}/gpdb/$file" ]; then - echo "✓ $file exists" - ls -l "${CGROUP_BASEDIR}/gpdb/$file" - else - echo "✗ $file missing" - fi - done - - # 8. Test subdirectory creation - echo "Testing subdirectory creation..." - sudo -u gpadmin bash -c " - TEST_DIR=\${CGROUP_BASEDIR}/gpdb/test6448 - if mkdir -p \$TEST_DIR; then - echo 'Created test directory' - sudo chmod -R 777 \$TEST_DIR - if echo \$\$ > \$TEST_DIR/cgroup.procs; then - echo 'Successfully wrote to cgroup.procs' - cat \$TEST_DIR/cgroup.procs - # Move processes back to parent before cleanup - echo \$\$ > \${CGROUP_BASEDIR}/gpdb/cgroup.procs - else - echo 'Failed to write to cgroup.procs' - ls -la \$TEST_DIR/cgroup.procs - fi - ls -la \$TEST_DIR/ - rmdir \$TEST_DIR || { - echo 'Moving all processes to parent before cleanup' - cat \$TEST_DIR/cgroup.procs | while read pid; do - echo \$pid > \${CGROUP_BASEDIR}/gpdb/cgroup.procs 2>/dev/null || true - done - rmdir \$TEST_DIR - } - else - echo 'Failed to create test directory' - fi - " - - # 9. Verify setup as gpadmin user - echo "Testing cgroup access as gpadmin..." - sudo -u gpadmin bash -c " - echo 'Checking mounts...' - mount | grep cgroup - - echo 'Checking /proc/self/mounts...' - cat /proc/self/mounts | grep cgroup - - if ! grep -q cgroup2 /proc/self/mounts; then - echo 'ERROR: cgroup2 mount NOT visible to gpadmin' - exit 1 - fi - echo 'SUCCESS: cgroup2 mount visible to gpadmin' - - if ! [ -w ${CGROUP_BASEDIR}/gpdb ]; then - echo 'ERROR: gpadmin cannot write to gpdb cgroup' - exit 1 - fi - echo 'SUCCESS: gpadmin can write to gpdb cgroup' - - echo 'Verifying key files content:' - echo 'cpu.max:' - cat ${CGROUP_BASEDIR}/gpdb/cpu.max || echo 'Failed to read cpu.max' - echo 'cpuset.cpus:' - cat ${CGROUP_BASEDIR}/gpdb/cpuset.cpus || echo 'Failed to read cpuset.cpus' - echo 'cgroup.subtree_control:' - cat ${CGROUP_BASEDIR}/gpdb/cgroup.subtree_control || echo 'Failed to read cgroup.subtree_control' - " - - # 10. Show final state - echo "Final cgroup state:" - ls -la ${CGROUP_BASEDIR}/gpdb/ - echo "Cgroup setup completed successfully" - else - echo "Cgroup setup skipped" - fi - - - name: "Generate Test Job Summary Start: ${{ matrix.test }}" - if: always() - run: | - { - echo "# Test Job Summary: ${{ matrix.test }} (Ubuntu 24.04)" - echo "## Environment" - echo "- Start Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - - if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then - echo "## Skip Status" - echo "✓ Test execution skipped via CI skip flag" - else - echo "- OS Version: $(cat /etc/redhat-release)" - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Download Cloudberry DEB build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-ubuntu24.04-build-artifacts - path: ${{ github.workspace }}/deb_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Download Cloudberry Source build artifacts - if: needs.check-skip.outputs.should_skip != 'true' - uses: actions/download-artifact@v4 - with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts - path: ${{ github.workspace }}/source_build_artifacts - merge-multiple: false - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Verify DEB artifacts - if: needs.check-skip.outputs.should_skip != 'true' - id: verify-artifacts - shell: bash - run: | - set -eo pipefail - - SRC_TARBALL_FILE=$(ls "${GITHUB_WORKSPACE}"/source_build_artifacts/apache-cloudberry-db-incubating_*.tar.xz) - if [ ! -f "${SRC_TARBALL_FILE}" ]; then - echo "::error::SRC TARBALL file not found" - exit 1 - fi - - echo "src_tarball_file=${SRC_TARBALL_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying SRC TARBALL artifacts..." - { - echo "=== SRC TARBALL Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "SRC TARBALL File: ${SRC_TARBALL_FILE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${SRC_TARBALL_FILE}" - - } 2>&1 | tee -a build-logs/details/src-tarball-verification.log - - DEB_FILE=$(ls "${GITHUB_WORKSPACE}"/deb_build_artifacts/*.deb) - if [ ! -f "${DEB_FILE}" ]; then - echo "::error::DEB file not found" - exit 1 - fi - - echo "deb_file=${DEB_FILE}" >> "$GITHUB_OUTPUT" - - echo "Verifying DEB artifacts..." - { - echo "=== DEB Verification Summary ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - - # Get DEB metadata and verify contents - echo "Package Information:" - dpkg-deb -f "${DEB_FILE}" - - # Get key DEB attributes for verification - DEB_VERSION=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 1) - DEB_RELEASE=$(dpkg-deb -f "${DEB_FILE}" Version | cut -d'-' -f 3) - echo "version=${DEB_VERSION}" >> "$GITHUB_OUTPUT" - echo "release=${DEB_RELEASE}" >> "$GITHUB_OUTPUT" - - # Verify expected binaries are in the DEB - echo "Verifying critical files in DEB..." - for binary in "bin/postgres" "bin/psql"; do - if ! dpkg-deb -c "${DEB_FILE}" | grep "${binary}" > /dev/null; then - echo "::error::Critical binary '${binary}' not found in DEB" - exit 1 - fi - done - - echo "DEB Details:" - echo "- Version: ${DEB_VERSION}" - echo "- Release: ${DEB_RELEASE}" - - # Calculate and store checksum - echo "Checksum:" - sha256sum "${DEB_FILE}" - - } 2>&1 | tee -a build-logs/details/deb-verification.log - - - name: Install Cloudberry DEB - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - DEB_FILE: ${{ steps.verify-artifacts.outputs.deb_file }} - DEB_VERSION: ${{ steps.verify-artifacts.outputs.version }} - DEB_RELEASE: ${{ steps.verify-artifacts.outputs.release }} - run: | - set -eo pipefail - - if [ -z "${DEB_FILE}" ]; then - echo "::error::DEB_FILE environment variable is not set" - exit 1 - fi - - { - echo "=== DEB Installation Log ===" - echo "Timestamp: $(date -u)" - echo "DEB File: ${DEB_FILE}" - echo "Version: ${DEB_VERSION}" - echo "Release: ${DEB_RELEASE}" - - # Clean install location - rm -rf /usr/local/cloudberry-db - - # Install DEB - echo "Starting installation..." - apt-get update - if ! apt-get -y install "${DEB_FILE}"; then - echo "::error::DEB installation failed" - exit 1 - fi - - # Change ownership back to gpadmin - it is needed for future tests - chown -R gpadmin:gpadmin /usr/local/cloudberry-db - - echo "Installation completed successfully" - dpkg-query -s apache-cloudberry-db-incubating - echo "Installed files:" - dpkg-query -L apache-cloudberry-db-incubating - } 2>&1 | tee -a build-logs/details/deb-installation.log - - - name: Extract source tarball - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_TARBALL_FILE: ${{ steps.verify-artifacts.outputs.src_tarball_file }} - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - echo "=== Source Extraction Log ===" - echo "Timestamp: $(date -u)" - - echo "Starting extraction..." - file "${SRC_TARBALL_FILE}" - if ! time tar xf "${SRC_TARBALL_FILE}" -C "${SRC_DIR}"/.. ; then - echo "::error::Source extraction failed" - exit 1 - fi - - echo "Extraction completed successfully" - echo "Extracted contents:" - ls -la "${SRC_DIR}/../cloudberry" - echo "Directory size:" - du -sh "${SRC_DIR}/../cloudberry" - } 2>&1 | tee -a build-logs/details/source-extraction.log - - - name: Prepare DEB Environment - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - - # change ownership to gpadmin - chown -R gpadmin "${SRC_DIR}/../cloudberry" - touch build-logs/sections.log - chown gpadmin build-logs/sections.log - chmod 777 build-logs - - # configure link lib directory to temporary location, fix it - rm -rf "${SRC_DIR}"/debian/build/lib - ln -sf /usr/cloudberry-db/lib "${SRC_DIR}"/debian/build/lib - - # check if regress.so exists in src directory - it is needed for contrib/dblink tests - if [ ! -f ${SRC_DIR}/src/test/regress/regress.so ]; then - ln -sf /usr/cloudberry-db/lib/postgresql/regress.so ${SRC_DIR}/src/test/regress/regress.so - fi - - # FIXME - # temporary install gdb - delete after creating new docker build/test contaners - apt-get update - apt-get -y install gdb - - } 2>&1 | tee -a build-logs/details/prepare-deb-env.log - - - name: Create Apache Cloudberry demo cluster - if: success() && needs.check-skip.outputs.should_skip != 'true' - shell: bash - env: - SRC_DIR: ${{ github.workspace }} - run: | - set -eo pipefail - - { - chmod +x "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh - - # Build BLDWRAP_POSTGRES_CONF_ADDONS for shared_preload_libraries if specified - EXTRA_CONF="" - if [[ -n "${{ matrix.shared_preload_libraries }}" ]]; then - EXTRA_CONF="shared_preload_libraries='${{ matrix.shared_preload_libraries }}'" - echo "Adding shared_preload_libraries: ${{ matrix.shared_preload_libraries }}" - fi - - if ! time su - gpadmin -c "cd ${SRC_DIR} && NUM_PRIMARY_MIRROR_PAIRS='${{ matrix.num_primary_mirror_pairs }}' BLDWRAP_POSTGRES_CONF_ADDONS=\"${EXTRA_CONF}\" SRC_DIR=${SRC_DIR} ${SRC_DIR}/devops/build/automation/cloudberry/scripts/create-cloudberry-demo-cluster.sh"; then - echo "::error::Demo cluster creation failed" - exit 1 - fi - - } 2>&1 | tee -a build-logs/details/create-cloudberry-demo-cluster.log - - - name: "Run Tests: ${{ matrix.test }}" - if: success() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - BUILD_DESTINATION: ${{ github.workspace }}/debian/build - shell: bash {0} - run: | - set -o pipefail - - # Initialize test status - overall_status=0 - - # Create logs directory structure - mkdir -p build-logs/details - - # Core file config - mkdir -p "/tmp/cloudberry-cores" - chmod 1777 "/tmp/cloudberry-cores" - sysctl -w kernel.core_pattern="/tmp/cloudberry-cores/core-%e-%s-%u-%g-%p-%t" - sysctl kernel.core_pattern - su - gpadmin -c "ulimit -c" - - # WARNING: PostgreSQL Settings - # When adding new pg_settings key/value pairs: - # 1. Add a new check below for the setting - # 2. Follow the same pattern as optimizer - # 3. Update matrix entries to include the new setting - - - # Create extension if required - if [[ "${{ matrix.extension != '' }}" == "true" ]]; then - case "${{ matrix.extension }}" in - gp_stats_collector) - if ! su - gpadmin -c "source ${BUILD_DESTINATION}/cloudberry-env.sh && \ - source ${SRC_DIR}/gpAux/gpdemo/gpdemo-env.sh && \ - gpconfig -c shared_preload_libraries -v 'gp_stats_collector' && \ - gpstop -ra && \ - echo 'CREATE EXTENSION IF NOT EXISTS gp_stats_collector; \ - SHOW shared_preload_libraries; \ - TABLE pg_extension;' | \ - psql postgres" - then - echo "Error creating gp_stats_collector extension" - exit 1 - fi - ;; - *) - echo "Unknown extension: ${{ matrix.extension }}" - exit 1 - ;; - esac - fi - - # Set PostgreSQL options if defined - PG_OPTS="" - if [[ "${{ matrix.pg_settings.optimizer != '' }}" == "true" ]]; then - PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}" - fi - - if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then - PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}" - fi - - # Read configs into array - IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" - - echo "=== Starting test execution for ${{ matrix.test }} ===" - echo "Number of configurations to execute: ${#configs[@]}" - echo "" - - # Execute each config separately - for ((i=0; i<${#configs[@]}; i++)); do - config="${configs[$i]}" - IFS=':' read -r dir target <<< "$config" - - echo "=== Executing configuration $((i+1))/${#configs[@]} ===" - echo "Make command: make -C $dir $target" - echo "Environment:" - echo "- PGOPTIONS: ${PG_OPTS}" - - # Create unique log file for this configuration - config_log="build-logs/details/make-${{ matrix.test }}-config$i.log" - - # Clean up any existing core files - echo "Cleaning up existing core files..." - rm -f /tmp/cloudberry-cores/core-* - - # Execute test script with proper environment setup - if ! time su - gpadmin -c "cd ${SRC_DIR} && \ - MAKE_NAME='${{ matrix.test }}-config$i' \ - MAKE_TARGET='$target' \ - MAKE_DIRECTORY='-C $dir' \ - PGOPTIONS='${PG_OPTS}' \ - SRC_DIR='${SRC_DIR}' \ - ${SRC_DIR}/devops/build/automation/cloudberry/scripts/test-cloudberry.sh" \ - 2>&1 | tee "$config_log"; then - echo "::warning::Test execution failed for configuration $((i+1)): make -C $dir $target" - overall_status=1 - fi - - # Check for results directory - results_dir="${dir}/results" - - if [[ -d "$results_dir" ]]; then - echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "Found results directory: $results_dir" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "Contents of results directory:" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - - find "$results_dir" -type f -ls >> "$log_file" 2>&1 | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - echo "-----------------------------------------" | tee -a build-logs/details/make-${{ matrix.test }}-config$i-results.log - else - echo "-----------------------------------------" - echo "Results directory $results_dir does not exit" - echo "-----------------------------------------" - fi - - # Analyze any core files generated by this test configuration - echo "Analyzing core files for configuration ${{ matrix.test }}-config$i..." - test_id="${{ matrix.test }}-config$i" - - # List the cores directory - echo "-----------------------------------------" - echo "Cores directory: /tmp/cloudberry-cores" - echo "Contents of cores directory:" - ls -Rl "/tmp/cloudberry-cores" - echo "-----------------------------------------" - - "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/analyze_core_dumps.sh "$test_id" - core_analysis_rc=$? - case "$core_analysis_rc" in - 0) echo "No core dumps found for this configuration" ;; - 1) echo "Core dumps were found and analyzed successfully" ;; - 2) echo "::warning::Issues encountered during core dump analysis" ;; - *) echo "::error::Unexpected return code from core dump analysis: $core_analysis_rc" ;; - esac - - echo "Log file: $config_log" - echo "=== End configuration $((i+1)) execution ===" - echo "" - done - - echo "=== Test execution completed ===" - echo "Log files:" - ls -l build-logs/details/ - - # Store number of configurations for parsing step - echo "NUM_CONFIGS=${#configs[@]}" >> "$GITHUB_ENV" - - # Report overall status - if [ $overall_status -eq 0 ]; then - echo "All test executions completed successfully" - else - echo "::warning::Some test executions failed, check individual logs for details" - fi - - exit $overall_status - - - name: "Parse Test Results: ${{ matrix.test }}" - id: test-results - if: always() && needs.check-skip.outputs.should_skip != 'true' - env: - SRC_DIR: ${{ github.workspace }} - shell: bash {0} - run: | - set -o pipefail - - overall_status=0 - - # Get configs array to create context for results - IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}" - - echo "=== Starting results parsing for ${{ matrix.test }} ===" - echo "Number of configurations to parse: ${#configs[@]}" - echo "" - - # Parse each configuration's results independently - for ((i=0; i "test_results.$i.txt" - overall_status=1 - continue - fi - - # Parse this configuration's results - - MAKE_NAME="${{ matrix.test }}-config$i" \ - "${SRC_DIR}"/devops/build/automation/cloudberry/scripts/parse-test-results.sh "$config_log" - status_code=$? - - { - echo "SUITE_NAME=${{ matrix.test }}" - echo "DIR=${dir}" - echo "TARGET=${target}" - } >> test_results.txt - - # Process return code - case $status_code in - 0) # All tests passed - echo "All tests passed successfully" - if [ -f test_results.txt ]; then - (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" - rm test_results.txt - fi - ;; - 1) # Tests failed but parsed successfully - echo "Test failures detected but properly parsed" - if [ -f test_results.txt ]; then - (echo "MAKE_COMMAND=\"make -C $dir $target\""; cat test_results.txt) | tee "test_results.${{ matrix.test }}.$i.txt" - rm test_results.txt - fi - overall_status=1 - ;; - 2) # Parse error or missing file - echo "::warning::Could not parse test results properly for configuration $((i+1))" - { - echo "MAKE_COMMAND=\"make -C $dir $target\"" - echo "STATUS=parse_error" - echo "TOTAL_TESTS=0" - echo "FAILED_TESTS=0" - echo "PASSED_TESTS=0" - echo "IGNORED_TESTS=0" - } | tee "test_results.${{ matrix.test }}.$i.txt" - overall_status=1 - ;; - *) # Unexpected error - echo "::warning::Unexpected error during test results parsing for configuration $((i+1))" - { - echo "MAKE_COMMAND=\"make -C $dir $target\"" - echo "STATUS=unknown_error" - echo "TOTAL_TESTS=0" - echo "FAILED_TESTS=0" - echo "PASSED_TESTS=0" - echo "IGNORED_TESTS=0" - } | tee "test_results.${{ matrix.test }}.$i.txt" - overall_status=1 - ;; - esac - - echo "Results stored in test_results.$i.txt" - echo "=== End parsing for configuration $((i+1)) ===" - echo "" - done - - # Report status of results files - echo "=== Results file status ===" - echo "Generated results files:" - for ((i=0; i> "$GITHUB_STEP_SUMMARY" || true - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - build-logs/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload Test Metadata - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-metadata-${{ matrix.test }} - path: | - test_results*.txt - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload test results files - uses: actions/upload-artifact@v4 - with: - name: results-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - **/regression.out - **/regression.diffs - **/results/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - - name: Upload test regression logs - if: failure() || cancelled() - uses: actions/upload-artifact@v4 - with: - name: regression-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} - path: | - **/regression.out - **/regression.diffs - **/results/ - gpAux/gpdemo/datadirs/standby/log/ - gpAux/gpdemo/datadirs/qddir/demoDataDir-1/log/ - gpAux/gpdemo/datadirs/dbfast1/demoDataDir0/log/ - gpAux/gpdemo/datadirs/dbfast2/demoDataDir1/log/ - gpAux/gpdemo/datadirs/dbfast3/demoDataDir2/log/ - gpAux/gpdemo/datadirs/dbfast_mirror1/demoDataDir0/log/ - gpAux/gpdemo/datadirs/dbfast_mirror2/demoDataDir1/log/ - gpAux/gpdemo/datadirs/dbfast_mirror3/demoDataDir2/log/ - retention-days: ${{ env.LOG_RETENTION_DAYS }} - - ## ====================================================================== - ## Job: report-deb - ## ====================================================================== - - report-deb: - name: Generate Apache Cloudberry Build Report (Ubuntu 24.04) - needs: [check-skip, build-deb, prepare-test-matrix-deb, deb-install-test, test-deb] - if: always() - runs-on: ubuntu-22.04 - steps: - - name: Generate Final Report - run: | - { - echo "# Apache Cloudberry Build Pipeline Report" - - if [[ "${{ needs.check-skip.outputs.should_skip }}" == "true" ]]; then - echo "## CI Skip Status" - echo "✅ CI checks skipped via skip flag" - echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - else - echo "## Job Status" - echo "- Build Job: ${{ needs.build-deb.result }}" - echo "- Test Job: ${{ needs.test-deb.result }}" - echo "- Completion Time: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - - if [[ "${{ needs.build-deb.result }}" == "success" && "${{ needs.test-deb.result }}" == "success" ]]; then - echo "✅ Pipeline completed successfully" - else - echo "⚠️ Pipeline completed with failures" - - if [[ "${{ needs.build-deb.result }}" != "success" ]]; then - echo "### Build Job Failure" - echo "Check build logs for details" - fi - - if [[ "${{ needs.test-deb.result }}" != "success" ]]; then - echo "### Test Job Failure" - echo "Check test logs and regression files for details" - fi - fi - fi - } >> "$GITHUB_STEP_SUMMARY" - - - name: Notify on failure - if: | - needs.check-skip.outputs.should_skip != 'true' && - (needs.build-deb.result != 'success' || needs.test-deb.result != 'success') - run: | - echo "::error::Build/Test pipeline failed! Check job summaries and logs for details" - echo "Timestamp: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" - echo "Build Result: ${{ needs.build-deb.result }}" - echo "Test Result: ${{ needs.test-deb.result }}" diff --git a/.github/workflows/build-deb-cloudberry.yml b/.github/workflows/build-deb-cloudberry.yml index f8eadee3c8f..630ce4438a4 100644 --- a/.github/workflows/build-deb-cloudberry.yml +++ b/.github/workflows/build-deb-cloudberry.yml @@ -80,6 +80,11 @@ on: types: [opened, synchronize, reopened, edited] workflow_dispatch: # Manual trigger inputs: + ubuntu_version: + description: 'Ubuntu version (default: all)' + required: false + default: 'all' + type: string test_selection: description: 'Select tests to run (comma-separated). Examples: ic-good-opt-off,ic-contrib' required: false @@ -280,6 +285,15 @@ jobs: get_defaults * .' } + # Determine ubuntu versions + UBUNTU_INPUT="${{ github.event.inputs.ubuntu_version }}" + if [[ "$UBUNTU_INPUT" == "all" || -z "$UBUNTU_INPUT" ]]; then + UBUNTU_VERSIONS_JSON='["22.04","24.04"]' + else + UBUNTU_VERSIONS_JSON=$(echo "$UBUNTU_INPUT" | jq -R 'split(",") | map(gsub("\\s+";"")) | map(select(length > 0)) | unique') + fi + echo "Ubuntu versions: $UBUNTU_VERSIONS_JSON" + # Extract all valid test names from ALL_TESTS VALID_TESTS=$(echo "$ALL_TESTS" | jq -r '.include[].test') @@ -325,14 +339,21 @@ jobs: done RESULT="${RESULT}]}" + # Expand by ubuntu versions: cross-join each test with each ubuntu version + FINAL_RESULT=$(jq -n \ + --argjson tests "$RESULT" \ + --argjson versions "$UBUNTU_VERSIONS_JSON" \ + '[ $tests.include[] as $t | $versions[] as $v | $t + {ubuntu_version: $v} ] | {include: .}') + # Output the matrix for GitHub Actions - echo "Final matrix configuration:" - echo "$RESULT" | jq . + echo "Final matrix configuration (first 3 entries):" + echo "$FINAL_RESULT" | jq '.include[:3]' + echo "Total matrix entries: $(echo "$FINAL_RESULT" | jq '.include | length')" # Fix: Use block redirection { echo "matrix<> "$GITHUB_OUTPUT" @@ -343,7 +364,7 @@ jobs: ## ====================================================================== build-deb: - name: Build Apache Cloudberry DEB + name: ${{ matrix.ubuntu_version == '22.04' && 'Build Apache Cloudberry DEB' || format('Build Apache Cloudberry DEB (Ubuntu {0})', matrix.ubuntu_version) }} env: JOB_TYPE: build needs: [check-skip] @@ -353,8 +374,13 @@ jobs: outputs: build_timestamp: ${{ steps.set_timestamp.outputs.timestamp }} + strategy: + fail-fast: false + matrix: + ubuntu_version: ['22.04', '24.04'] + container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + image: apache/incubator-cloudberry:cbdb-build-ubuntu${{ matrix.ubuntu_version }}-latest options: >- --user root -h cdw @@ -542,7 +568,7 @@ jobs: echo "=== Artifact Creation Log ===" echo "Timestamp: $(date -u)" - cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu22.04/* debian/ + cp -r "${SRC_DIR}"/devops/build/packaging/deb/ubuntu${{ matrix.ubuntu_version }}/* debian/ chown -R "$(whoami)" debian chmod -x debian/*install @@ -634,7 +660,7 @@ jobs: - name: Upload build logs uses: actions/upload-artifact@v4 with: - name: build-logs-${{ env.BUILD_TIMESTAMP }} + name: build-logs-ubuntu${{ matrix.ubuntu_version }}-${{ env.BUILD_TIMESTAMP }} path: | build-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -642,7 +668,7 @@ jobs: - name: Upload Cloudberry DEB build artifacts uses: actions/upload-artifact@v4 with: - name: apache-cloudberry-db-incubating-deb-build-artifacts + name: apache-cloudberry-db-incubating-deb-build-artifacts-ubuntu${{ matrix.ubuntu_version }} retention-days: ${{ env.LOG_RETENTION_DAYS }} if-no-files-found: error path: | @@ -652,7 +678,7 @@ jobs: - name: Upload Cloudberry deb source build artifacts uses: actions/upload-artifact@v4 with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts + name: apache-cloudberry-db-incubating-deb-source-build-artifacts-ubuntu${{ matrix.ubuntu_version }} retention-days: ${{ env.LOG_RETENTION_DAYS }} if-no-files-found: error path: | @@ -667,7 +693,7 @@ jobs: ## ====================================================================== deb-install-test: - name: DEB Install Test Apache Cloudberry + name: ${{ matrix.ubuntu_version == '22.04' && 'DEB Install Test Apache Cloudberry' || format('DEB Install Test Apache Cloudberry (Ubuntu {0})', matrix.ubuntu_version) }} needs: [check-skip, build-deb] if: | !cancelled() && @@ -676,8 +702,13 @@ jobs: runs-on: ubuntu-22.04 timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + ubuntu_version: ['22.04', '24.04'] + container: - image: apache/incubator-cloudberry:cbdb-test-ubuntu22.04-latest + image: apache/incubator-cloudberry:cbdb-test-ubuntu${{ matrix.ubuntu_version }}-latest options: >- --user root -h cdw @@ -718,10 +749,11 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' uses: actions/download-artifact@v4 with: - name: apache-cloudberry-db-incubating-deb-build-artifacts + name: apache-cloudberry-db-incubating-deb-build-artifacts-ubuntu${{ matrix.ubuntu_version }} path: ${{ github.workspace }}/deb_build_artifacts - run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} merge-multiple: false + run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Cloudberry Environment Initialization if: needs.check-skip.outputs.should_skip != 'true' @@ -850,7 +882,7 @@ jobs: - name: Upload install logs uses: actions/upload-artifact@v4 with: - name: install-logs-${{ matrix.name }}-${{ needs.build-deb.outputs.build_timestamp }} + name: install-logs-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | install-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -872,7 +904,7 @@ jobs: ## ====================================================================== test-deb: - name: ${{ matrix.test }} + name: ${{ matrix.ubuntu_version == '22.04' && matrix.test || format('{0} (Ubuntu {1})', matrix.test, matrix.ubuntu_version) }} needs: [check-skip, build-deb, prepare-test-matrix-deb] if: | !cancelled() && @@ -885,7 +917,7 @@ jobs: matrix: ${{ fromJson(needs.prepare-test-matrix-deb.outputs.test-matrix) }} container: - image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest + image: apache/incubator-cloudberry:cbdb-build-ubuntu${{ matrix.ubuntu_version }}-latest options: >- --privileged --user root @@ -1129,7 +1161,7 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' uses: actions/download-artifact@v4 with: - name: apache-cloudberry-db-incubating-deb-build-artifacts + name: apache-cloudberry-db-incubating-deb-build-artifacts-ubuntu${{ matrix.ubuntu_version }} path: ${{ github.workspace }}/deb_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -1139,7 +1171,7 @@ jobs: if: needs.check-skip.outputs.should_skip != 'true' uses: actions/download-artifact@v4 with: - name: apache-cloudberry-db-incubating-deb-source-build-artifacts + name: apache-cloudberry-db-incubating-deb-source-build-artifacts-ubuntu${{ matrix.ubuntu_version }} path: ${{ github.workspace }}/source_build_artifacts merge-multiple: false run-id: ${{ github.event.inputs.reuse_artifacts_from_run_id || github.run_id }} @@ -1788,7 +1820,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + name: test-logs-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | build-logs/ retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -1797,7 +1829,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: test-metadata-${{ matrix.test }} + name: test-metadata-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }} path: | test_results*.txt retention-days: ${{ env.LOG_RETENTION_DAYS }} @@ -1805,7 +1837,7 @@ jobs: - name: Upload test results files uses: actions/upload-artifact@v4 with: - name: results-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + name: results-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | **/regression.out **/regression.diffs @@ -1816,7 +1848,7 @@ jobs: if: failure() || cancelled() uses: actions/upload-artifact@v4 with: - name: regression-logs-${{ matrix.test }}-${{ needs.build-deb.outputs.build_timestamp }} + name: regression-logs-${{ matrix.test }}-ubuntu${{ matrix.ubuntu_version }}-${{ needs.build-deb.outputs.build_timestamp }} path: | **/regression.out **/regression.diffs