From 11983b56a5cb8e42967056a20aafa339ec716631 Mon Sep 17 00:00:00 2001 From: Jared Casey Date: Tue, 5 Aug 2025 10:47:11 -0600 Subject: [PATCH] Update Github Workflows Changes ======= * Update tests workflow to use linux arm runners * Split alpine tests into separate job (and cache docker image) * PYCO-56 & PYCO-63: Add publish workflow to handle releasing artifacts and API docs * PYCO-64: Add verify_release workflow to download published versions and run tests for validation --- .github/workflows/publish.yml | 378 ++++++++++++++++++ .github/workflows/tests.yml | 307 ++++++++++++--- .github/workflows/verify_release.yml | 566 +++++++++++++++++++++++++++ 3 files changed, 1190 insertions(+), 61 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/verify_release.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..a2167eb --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,378 @@ +name: publish + +on: + workflow_dispatch: + inputs: + sha: + description: "The git SHA to use for release. Only set if needing to publish" + required: true + default: "" + type: string + version: + description: "The Release version. Allowed format: x.y.z[-alphaN | -betaN | -rcN | -devN | -postN]" + required: true + default: "" + type: string + tests_run_id: + description: "The workflow run ID of a tests workflow run. Set if wanting to use artifacts from an already completed run." + required: false + default: "" + type: string + config: + description: "JSON formatted object representing various build system input parameters." + required: false + default: "" + type: string + is_dry_run: + description: "Set to true in order to do a publish dry-run." + required: true + default: false + type: boolean + skip_validation: + description: "Set to true in order to skip validating the published packages." + required: true + default: false + type: boolean + +env: + CBCI_PROJECT_TYPE: "ANALYTICS" + CBCI_DEFAULT_PYTHON: "3.9" + CBCI_SUPPORTED_PYTHON_VERSIONS: "3.9 3.10 3.11 3.12 3.13" + CBCI_SUPPORTED_X86_64_PLATFORMS: "linux alpine macos windows" + CBCI_SUPPORTED_ARM64_PLATFORMS: "linux macos" + CBCI_DEFAULT_LINUX_X86_64_PLATFORM: "ubuntu-22.04" + CBCI_DEFAULT_LINUX_ARM64_PLATFORM: "ubuntu-22.04-arm" + CBCI_DEFAULT_MACOS_X86_64_PLATFORM: "macos-13" + CBCI_DEFAULT_MACOS_ARM64_PLATFORM: "macos-14" + CBCI_DEFAULT_WINDOWS_PLATFORM: "windows-2022" + CBCI_DEFAULT_LINUX_CONTAINER: "slim-bookworm" + CBCI_DEFAULT_ALPINE_CONTAINER: "alpine" + CBCI_CBDINO_VERSION: "v0.0.80" + CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python/ci_scripts_v1" + +jobs: + default-linux-runner: + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.set_runner.outputs.runner }} + steps: + - id: set_runner + run: echo "runner=${{ env.CBCI_DEFAULT_LINUX_X86_64_PLATFORM }}" >> $GITHUB_OUTPUT + + ci-scripts: + needs: default-linux-runner + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + steps: + - name: Download CI Scripts + run: | + mkdir ci_scripts + cd ci_scripts + curl -o gha.sh ${CI_SCRIPTS_URL}/gha.sh + curl -o pygha.py ${CI_SCRIPTS_URL}/pygha.py + ls -alh + - name: Upload CI scripts + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: ci_scripts_publish + path: | + ci_scripts/ + + validate-input: + needs: [ci-scripts, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + env: + CBCI_IS_RELEASE: true + CBCI_SHA: ${{ inputs.sha }} + CBCI_VERSION: ${{ inputs.version }} + CBCI_CONFIG: ${{ inputs.config }} + steps: + - name: Download CI scripts + uses: actions/download-artifact@v4 + with: + name: ci_scripts_publish + path: ci_scripts + - name: Verify Scripts + run: | + ls -alh ci_scripts + chmod +x ci_scripts/gha.sh + ls -alh ci_scripts + - name: Display workflow info + run: | + ./ci_scripts/gha.sh display_info + - name: Validate workflow info + run: | + ./ci_scripts/gha.sh validate_input ${{ github.workflow }} + + publish-config: + needs: [validate-input, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + env: + CBCI_IS_RELEASE: true + CBCI_SHA: ${{ inputs.sha }} + CBCI_VERSION: ${{ inputs.version }} + CBCI_CONFIG: ${{ inputs.config }} + CBCI_PUBLISH_DRY_RUN: ${{ inputs.is_dry_run }} + CBCI_TESTS_RUN_ID: ${{ inputs.tests_run_id }} + outputs: + publish_config: ${{ steps.set_config.outputs.publish_config }} + steps: + - name: Download CI scripts + uses: actions/download-artifact@v4 + with: + name: ci_scripts_publish + path: ci_scripts + - name: Enable CI Scripts + run: | + chmod +x ci_scripts/gha.sh + - name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CBCI_DEFAULT_PYTHON }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Set public config + id: set_config + run: | + exit_code=0 + PUBLISH_CONFIG=$(./ci_scripts/gha.sh build_publish_config) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Failed to obtain publish config." + exit 1 + fi + publish_config_json=$(jq -cn --argjson pubconfig "$PUBLISH_CONFIG" '$pubconfig') + echo "PUBLISH_CONFIG_JSON=$publish_config_json" + echo "publish_config=$publish_config_json" >> "$GITHUB_OUTPUT" + + confirm-publish-config: + needs: [publish-config, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + steps: + - name: Publish Config + run: | + echo "${{ toJson(fromJson(needs.publish-config.outputs.publish_config)) }}" + + run-tests: + needs: publish-config + if: ${{ fromJson(needs.publish-config.outputs.publish_config).publish_config.tests_run_id == '' }} + uses: ./.github/workflows/tests.yml + with: + is_release: true + sha: ${{ inputs.sha && inputs.sha || '' }} + version: ${{ inputs.version && inputs.version || '' }} + config: ${{ inputs.config && inputs.config || '' }} + + set-tests-run-id: + needs: [run-tests, default-linux-runner] + if: >- + ${{ !cancelled() + && (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') }} + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + outputs: + tests_run_id: ${{ inputs.tests_run_id == '' && steps.save_tests_run_id.outputs.tests_run_id || inputs.tests_run_id }} + steps: + - name: Save tests_run_id + if: ${{ inputs.tests_run_id == '' }} + id: save_tests_run_id + run: | + echo "tests_run_id=${{ needs.run-tests.outputs.workflow_run_id }}" >> $GITHUB_OUTPUT + + upload-api-docs: + needs: [publish-config, run-tests, default-linux-runner] + if: >- + ${{ !cancelled() + && fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_api_docs + && (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') }} + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + steps: + - name: Checkout (with SHA) + if: inputs.sha != '' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 + - name: Checkout (no SHA) + if: inputs.sha == '' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CBCI_DEFAULT_PYTHON }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Set SDK git tag + if: ${{ fromJson(needs.publish-config.outputs.publish_config).publish_config.set_git_tag != '' }} + run: | + git config user.name "Couchbase SDK Team" + git config user.email "sdk_dev@couchbase.com" + git tag -a $PYCBAC_VERSION -m "Release of client version $PYCBAC_VERSION" + env: + PYCBAC_VERSION: ${{ fromJson(needs.publish-config.outputs.publish_config).publish_config.set_git_tag }} + - name: Set SDK version + run: | + python couchbase_analytics_version.py --mode make + - name: Build API docs + run: | + python -m pip install --upgrade pip setuptools wheel + python -m pip install -r requirements.txt + python -m pip install -r requirements-sphinx.txt + mkdir sphinx + sphinx-build -M html ./docs ./sphinx + - name: Upload Python API docs as artifact + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: pycbac-api-docs + path: sphinx/ + + publish-test-pypi: + needs: [publish-config, run-tests, set-tests-run-id, default-linux-runner] + if: >- + ${{ !cancelled() + && fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_test_pypi + && (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') }} + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + environment: + name: publish + url: https://pypi.org/p/couchbase-analytics + permissions: + # This permission is required for pypi's "trusted publisher" feature + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: pycbac-artifact-* + path: dist + merge-multiple: true + run-id: ${{ needs.set-tests-run-id.outputs.tests_run_id }} + github-token: ${{ github.token }} + - name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CBCI_DEFAULT_PYTHON }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel twine + - name: Twine validate + run: | + ls -alh dist + twine check dist/* + - name: Publish package distributions to TestPyPI (dry run) + if: ${{ inputs.is_dry_run }} + run: | + echo "Dry run set, not releasing to TestPyPI" + - name: Publish package distributions to TestPyPI + if: ${{ inputs.is_dry_run == false }} + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + + validate-test-pypi: + needs: publish-test-pypi + if: >- + ${{ !cancelled() + && (needs.publish-test-pypi.result == 'success' && needs.publish-test-pypi.result != 'skipped') + && inputs.skip_validation == false }} + uses: ./.github/workflows/verify_release.yml + with: + version: ${{ inputs.version }} + sha: ${{ inputs.sha }} + packaging_index: "TEST_PYPI" + config: ${{ inputs.config && inputs.config || '' }} + + publish-pypi: + needs: [publish-config, run-tests, set-tests-run-id, validate-test-pypi, default-linux-runner] + if: >- + ${{ !cancelled() + && fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_pypi + && (needs.run-tests.result == 'success' || needs.run-tests.result == 'skipped') + && (needs.validate-test-pypi.result == 'success' || needs.validate-test-pypi.result == 'skipped') }} + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + environment: + name: publish + url: https://pypi.org/p/couchbase-analytics + permissions: + # This permission is required for pypi's "trusted publisher" feature + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: pycbac-artifact-* + path: dist + merge-multiple: true + run-id: ${{ needs.set-tests-run-id.outputs.tests_run_id }} + github-token: ${{ github.token }} + - name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CBCI_DEFAULT_PYTHON }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel twine + - name: Twine validate + run: | + ls -alh dist + twine check dist/* + - name: Publish package distributions to PyPI (dry run) + if: ${{ inputs.is_dry_run }} + run: | + echo "Dry run set, not releasing to PyPI" + - name: Publish package distributions to PyPI + if: ${{ inputs.is_dry_run == false }} + uses: pypa/gh-action-pypi-publish@release/v1 + + validate-pypi: + needs: publish-pypi + if: >- + ${{ !cancelled() + && (needs.publish-pypi.result == 'success' && needs.publish-pypi.result != 'skipped') + && inputs.skip_validation == false }} + uses: ./.github/workflows/verify_release.yml + with: + version: ${{ inputs.version }} + sha: ${{ inputs.sha }} + packaging_index: "PYPI" + config: ${{ inputs.config && inputs.config || '' }} + + publish-api-docs: + needs: [upload-api-docs, validate-pypi, default-linux-runner] + if: >- + ${{ !cancelled() + && fromJson(needs.publish-config.outputs.publish_config).publish_config.publish_api_docs + && (needs.validate-pypi.result == 'success' || needs.validate-pypi.result == 'skipped') }} + environment: publish + permissions: + id-token: write + contents: read + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + steps: + - name: Download API docs + uses: actions/download-artifact@v4 + with: + name: pycbac-api-docs + - name: Setup AWS Credentials + if: ${{ inputs.is_dry_run == false }} + uses: aws-actions/configure-aws-credentials@v3 + with: + role-to-assume: arn:aws:iam::786014483886:role/SDK_GHA + aws-region: us-west-1 + - name: Upload API docs to S3 + if: ${{ inputs.is_dry_run == false }} + run: | + ls -alh ./html + aws s3 cp ./html s3://docs.couchbase.com/sdk-api/analytics-python-client-$SDK_VERSION/ --recursive --acl public-read + env: + SDK_VERSION: ${{ inputs.version }} + - name: Upload API docs to S3 (dry run) + if: ${{ inputs.is_dry_run }} + run: | + ls -alh ./html + echo "aws s3 cp ./html s3://docs.couchbase.com/sdk-api/analytics-python-client-$SDK_VERSION/ --recursive --acl public-read" + env: + SDK_VERSION: ${{ inputs.version }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cfddc67..6366d78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,11 +4,9 @@ on: push: branches: - main - - dev pull_request: branches: - main - - dev workflow_dispatch: inputs: is_release: @@ -64,7 +62,8 @@ env: CBCI_SUPPORTED_PYTHON_VERSIONS: "3.9 3.10 3.11 3.12 3.13" CBCI_SUPPORTED_X86_64_PLATFORMS: "linux alpine macos windows" CBCI_SUPPORTED_ARM64_PLATFORMS: "linux macos" - CBCI_DEFAULT_LINUX_PLATFORM: "ubuntu-22.04" + CBCI_DEFAULT_LINUX_X86_64_PLATFORM: "ubuntu-22.04" + CBCI_DEFAULT_LINUX_ARM64_PLATFORM: "ubuntu-22.04-arm" CBCI_DEFAULT_MACOS_X86_64_PLATFORM: "macos-13" CBCI_DEFAULT_MACOS_ARM64_PLATFORM: "macos-14" CBCI_DEFAULT_WINDOWS_PLATFORM: "windows-2022" @@ -74,8 +73,17 @@ env: CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python/ci_scripts_v1" jobs: + default-linux-runner: + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.set_runner.outputs.runner }} + steps: + - id: set_runner + run: echo "runner=${{ env.CBCI_DEFAULT_LINUX_X86_64_PLATFORM }}" >> $GITHUB_OUTPUT + ci-scripts: - runs-on: ubuntu-22.04 + needs: default-linux-runner + runs-on: ${{ needs.default-linux-runner.outputs.runner }} steps: - name: Download CI Scripts run: | @@ -88,13 +96,13 @@ jobs: uses: actions/upload-artifact@v4 with: retention-days: 1 - name: ci_scripts + name: ci_scripts_tests path: | ci_scripts/ validate-input: - runs-on: ubuntu-22.04 - needs: ci-scripts + needs: [ci-scripts, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} env: CBCI_IS_RELEASE: ${{ inputs.is_release }} CBCI_SHA: ${{ inputs.sha }} @@ -104,7 +112,7 @@ jobs: - name: Download CI scripts uses: actions/download-artifact@v4 with: - name: ci_scripts + name: ci_scripts_tests path: ci_scripts - name: Verify Scripts run: | @@ -119,18 +127,17 @@ jobs: ./ci_scripts/gha.sh validate_input ${{ github.workflow }} setup: - runs-on: ubuntu-22.04 - needs: validate-input + needs: [validate-input, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} env: CBCI_CONFIG: ${{ inputs.config }} outputs: stage_matrices: ${{ steps.build_matrices.outputs.stage_matrices }} steps: - - uses: actions/checkout@v4 - name: Download CI scripts uses: actions/download-artifact@v4 with: - name: ci_scripts + name: ci_scripts_tests path: ci_scripts - name: Enable CI Scripts run: | @@ -155,9 +162,13 @@ jobs: echo "stage_matrices=$stage_matrices_json" >> "$GITHUB_OUTPUT" confirm-matrices: - runs-on: ubuntu-22.04 - needs: setup + needs: [setup, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} steps: + - name: Alpine Test Unit Stage + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_alpine }} + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.alpine) }}" - name: Linux Test Unit Stage if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_linux }} run: | @@ -170,6 +181,12 @@ jobs: if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_windows }} run: | echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.windows) }}" + - name: Unit Test Commands + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest) }}" + - name: Integration Test Commands + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest) }}" - name: Linux cbdino Stage if: >- ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.has_linux_cbdino @@ -186,18 +203,24 @@ jobs: run: | echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_integration.test_config) }}" - test-setup: - runs-on: ubuntu-22.04 - needs: confirm-matrices + needs: [confirm-matrices, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} env: CBCI_CONFIG: ${{ inputs.config }} steps: - - uses: actions/checkout@v4 + - name: Checkout (with SHA) + if: inputs.sha != '' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + - name: Checkout (no SHA) + if: inputs.sha == '' + uses: actions/checkout@v4 - name: Download CI scripts uses: actions/download-artifact@v4 with: - name: ci_scripts + name: ci_scripts_tests path: ci_scripts - name: Enable CI Scripts run: | @@ -238,16 +261,23 @@ jobs: pycbac_test/ lint: - runs-on: ubuntu-22.04 - needs: validate-input + needs: [validate-input, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} env: CBCI_VERSION: ${{ inputs.version }} steps: - - uses: actions/checkout@v4 + - name: Checkout (with SHA) + if: inputs.sha != '' + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + - name: Checkout (no SHA) + if: inputs.sha == '' + uses: actions/checkout@v4 - name: Download CI scripts uses: actions/download-artifact@v4 with: - name: ci_scripts + name: ci_scripts_tests path: ci_scripts - name: Enable CI Scripts run: | @@ -264,8 +294,8 @@ jobs: ./ci_scripts/gha.sh lint sdist-wheel: - runs-on: ubuntu-22.04 - needs: lint + needs: [lint, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} env: CBCI_VERSION: ${{ inputs.version }} CBCI_CONFIG: ${{ inputs.config }} @@ -284,7 +314,7 @@ jobs: - name: Download CI scripts uses: actions/download-artifact@v4 with: - name: ci_scripts + name: ci_scripts_tests path: ci_scripts - name: Enable CI Scripts run: | @@ -329,14 +359,13 @@ jobs: path: | ./dist/*.whl - linux-unit-tests: - needs: [setup, test-setup, sdist-wheel] - if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_linux }} - name: Run unit tests; Python ${{ matrix.python-version }} - ${{ matrix.linux-type }} (${{ matrix.arch }}) - runs-on: ubuntu-22.04 + alpine-unit-tests: + needs: [setup, test-setup, sdist-wheel, default-linux-runner] + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_alpine }} + runs-on: ${{ needs.default-linux-runner.outputs.runner }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.linux }} + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.alpine }} steps: - name: Setup Python uses: actions/setup-python@v5 @@ -344,11 +373,6 @@ jobs: python-version: ${{ matrix.python-version }} - name: Confirm Python version run: python -c "import sys; print(sys.version)" - - name: Set up QEMU - if: ${{ matrix.arch == 'aarch64' }} - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - name: Download sdist if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_sdist_install }} uses: actions/download-artifact@v4 @@ -366,16 +390,31 @@ jobs: with: name: pycbac-test-setup path: pycbac + - name: Cache Docker image + uses: actions/cache@v4 + with: + path: /tmp/docker_python_image.tar # Path to store the image archive + key: docker-image-python-${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} + - name: Pull and save image if not cached + run: | + if [ ! -f /tmp/docker_python_image.tar ]; then + docker pull python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} + docker save python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} -o /tmp/docker_python_image.tar + fi + - name: Load image if cached + run: | + if [ -f /tmp/docker_python_image.tar ]; then + docker load -i /tmp/docker_python_image.tar + fi - name: Run unit tests in docker via sdist install if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_sdist_install }} uses: addnab/docker-run-action@v3 with: - image: python:${{ matrix.python-version }}-${{ matrix.linux-type == 'manylinux' && env.CBCI_DEFAULT_LINUX_CONTAINER || env.CBCI_DEFAULT_ALPINE_CONTAINER }} + image: python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} options: >- - --platform linux/${{ matrix.arch == 'aarch64' && 'arm64' || 'amd64'}} + --platform linux/amd64 -v ${{ github.workspace }}/pycbac:/pycbac run: | - apt-get update && apt-get install -y jq python -m pip install --upgrade pip setuptools wheel cd pycbac ls -alh @@ -386,22 +425,29 @@ jobs: python -m pip list TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} if [ "$TEST_ACOUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_acouchbase and pycbac_unit" -rA -vv + PYTEST_ACB_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }}" + PYTEST_ACB_OPTS="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }}" + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} if [ "$TEST_COUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_couchbase and pycbac_unit" -rA -vv + PYTEST_CB_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }}" + PYTEST_CB_OPTS="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }}" + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi - name: Run unit tests in docker via wheel install if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_wheel_install }} uses: addnab/docker-run-action@v3 with: - image: python:${{ matrix.python-version }}-${{ matrix.linux-type == 'manylinux' && env.CBCI_DEFAULT_LINUX_CONTAINER || env.CBCI_DEFAULT_ALPINE_CONTAINER }} + image: python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} options: >- - --platform linux/${{ matrix.arch == 'aarch64' && 'arm64' || 'amd64'}} + --platform linux/amd64 -v ${{ github.workspace }}/pycbac:/pycbac run: | - apt-get update && apt-get install -y jq python -m pip install --upgrade pip setuptools wheel cd pycbac ls -alh @@ -412,17 +458,113 @@ jobs: python -m pip list TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} if [ "$TEST_ACOUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_acouchbase and pycbac_unit" -rA -vv + PYTEST_ACB_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }}" + PYTEST_ACB_OPTS="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }}" + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} if [ "$TEST_COUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_couchbase and pycbac_unit" -rA -vv + PYTEST_CB_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }}" + PYTEST_CB_OPTS="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }}" + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi + linux-unit-tests: + needs: [setup, test-setup, sdist-wheel] + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_linux }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.linux }} + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download sdist + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_sdist_install }} + uses: actions/download-artifact@v4 + with: + name: pycbac-artifact-sdist + path: pycbac + - name: Download wheel + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_wheel_install }} + uses: actions/download-artifact@v4 + with: + name: pycbac-artifact-wheel + path: pycbac + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbac-test-setup + path: pycbac + - name: Run unit tests via sdist install + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_sdist_install }} + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + ls -alh + python -m pip install -r requirements-test.txt + SDIST_NAME=${{ needs.sdist-wheel.outputs.sdist_name }} + echo "SDIST_NAME=$SDIST_NAME.tar.gz" + python -m pip install ${SDIST_NAME}.tar.gz + python -m pip list + TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} + if [ "$TEST_ACOUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} + if [ "$TEST_COUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + python -m pip uninstall couchbase-analytics -y + - name: Run unit tests via wheel install + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_wheel_install }} + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + ls -alh + python -m pip install -r requirements-test.txt + WHEEL_NAME=${{ needs.sdist-wheel.outputs.wheel_name }} + echo "WHEEL_NAME=$WHEEL_NAME" + python -m pip install ${WHEEL_NAME} --no-cache-dir + python -m pip list + TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} + if [ "$TEST_ACOUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} + if [ "$TEST_COUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + macos-unit-tests: needs: [setup, test-setup, sdist-wheel] if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_macos }} - name: Run unit tests; Python ${{ matrix.python-version }} - ${{ matrix.arch }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -453,6 +595,11 @@ jobs: path: pycbac - name: Run unit tests via sdist install if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_sdist_install }} + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} run: | python -m pip install --upgrade pip setuptools wheel cd pycbac @@ -464,15 +611,24 @@ jobs: python -m pip list TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} if [ "$TEST_ACOUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_acouchbase and pycbac_unit" -rA -vv --log-cli-level=DEBUG + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} if [ "$TEST_COUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_couchbase and pycbac_unit" -rA -vv --log-cli-level=DEBUG + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi python -m pip uninstall couchbase-analytics -y - name: Run unit tests via wheel install if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_wheel_install }} + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} run: | python -m pip install --upgrade pip setuptools wheel cd pycbac @@ -484,17 +640,20 @@ jobs: python -m pip list TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} if [ "$TEST_ACOUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_acouchbase and pycbac_unit" -rA -vv --log-cli-level=DEBUG + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} if [ "$TEST_COUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_couchbase and pycbac_unit" -rA -vv --log-cli-level=DEBUG + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi windows-unit-tests: needs: [setup, test-setup, sdist-wheel] if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_windows }} - name: Run unit tests; Python ${{ matrix.python-version }} - ${{ matrix.arch }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -525,6 +684,11 @@ jobs: path: pycbac - name: Run unit tests via sdist install if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_sdist_install }} + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} run: | python -m pip install --upgrade pip setuptools wheel cd pycbac @@ -536,15 +700,24 @@ jobs: python -m pip list $TEST_ACOUCHBASE_API="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }}" if ( $TEST_ACOUCHBASE_API -eq "true" ) { - python -m pytest -m "pycbac_acouchbase and pycbac_unit" -rA -vv + $PYTEST_CMD="python -m $($env:PYTEST_ACB_CMD) $($env:PYTEST_ACB_OPTS)" + echo "Running acouchbase tests: $PYTEST_CMD" + iex $PYTEST_CMD } $TEST_COUCHBASE_API="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }}" if ( $TEST_COUCHBASE_API = "true" ) { - python -m pytest -m "pycbac_couchbase and pycbac_unit" -rA -vv + $PYTEST_CMD="python -m $($env:PYTEST_CB_CMD) $($env:PYTEST_CB_OPTS)" + echo "Running couchbase tests: $PYTEST_CMD" + iex $PYTEST_CMD } python -m pip uninstall couchbase-analytics -y - name: Run unit tests via wheel install if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_wheel_install }} + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} run: | python -m pip install --upgrade pip setuptools wheel cd pycbac @@ -556,11 +729,15 @@ jobs: python -m pip list $TEST_ACOUCHBASE_API="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }}" if ( $TEST_ACOUCHBASE_API -eq "true" ) { - python -m pytest -m "pycbac_acouchbase and pycbac_unit" -rA -vv + $PYTEST_CMD="python -m $($env:PYTEST_ACB_CMD) $($env:PYTEST_ACB_OPTS)" + echo "Running acouchbase tests: $PYTEST_CMD" + iex $PYTEST_CMD } $TEST_COUCHBASE_API="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }}" if ( $TEST_COUCHBASE_API = "true" ) { - python -m pytest -m "pycbac_couchbase and pycbac_unit" -rA -vv + $PYTEST_CMD="python -m $($env:PYTEST_CB_CMD) $($env:PYTEST_CB_OPTS)" + echo "Running couchbase tests: $PYTEST_CMD" + iex $PYTEST_CMD } cbdino-integration-tests: @@ -568,7 +745,6 @@ jobs: if: >- ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.has_linux_cbdino && !fromJson(needs.setup.outputs.stage_matrices).test_integration.skip_cbdino }} - name: Run integration tests w/ cbdino; Python ${{ matrix.python-version }} - ${{ matrix.os }} (${{ matrix.arch }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -600,7 +776,7 @@ jobs: - name: Download CI scripts uses: actions/download-artifact@v4 with: - name: ci_scripts + name: ci_scripts_tests path: ci_scripts - name: Enable CI Scripts run: | @@ -635,6 +811,11 @@ jobs: ./ci_scripts/gha.sh build_test_config_ini pycbac/tests - name: Run tests timeout-minutes: 30 + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.couchbase_opts }} run: | python -m pip install --upgrade pip setuptools wheel cd pycbac @@ -647,11 +828,15 @@ jobs: python -m pip list TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.test_acouchbase_api }} if [ "$TEST_ACOUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_acouchbase and pycbac_integration" -rA -vv + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.test_couchbase_api }} if [ "$TEST_COUCHBASE_API" = "true" ]; then - python -m pytest -m "pycbac_couchbase and pycbac_integration" -rA -vv + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" fi - name: Cleanup cbdino cluster run: | diff --git a/.github/workflows/verify_release.yml b/.github/workflows/verify_release.yml new file mode 100644 index 0000000..107817d --- /dev/null +++ b/.github/workflows/verify_release.yml @@ -0,0 +1,566 @@ +name: verify_release + +on: + workflow_dispatch: + inputs: + version: + description: "The Release version. Allowed format: x.y.z[-alphaN | -betaN | -rcN | -devN | -postN]" + required: true + default: "" + type: string + sha: + description: "The git SHA associated with the version to verify." + required: true + default: "" + type: string + packaging_index: + description: "The packaging index to download the SDK from." + required: true + default: "PYPI" + type: choice + options: + - PYPI + - TEST_PYPI + config: + description: "JSON formatted object representing various build system input parameters." + required: false + default: "" + type: string + workflow_call: + inputs: + version: + description: "The Release version. Allowed format: x.y.z[-alphaN | -betaN | -rcN | -devN | -postN]" + required: true + default: "" + type: string + sha: + description: "The git SHA associated with the version to verify." + required: true + default: "" + type: string + packaging_index: + description: "The packaging index to download the SDK from." + required: true + default: "PYPI" + type: string + config: + description: "JSON formatted object representing various build system input parameters." + required: false + default: "" + type: string + +env: + CBCI_PROJECT_TYPE: "ANALYTICS" + CBCI_DEFAULT_PYTHON: "3.9" + CBCI_SUPPORTED_PYTHON_VERSIONS: "3.9 3.10 3.11 3.12 3.13" + CBCI_SUPPORTED_X86_64_PLATFORMS: "linux alpine macos windows" + CBCI_SUPPORTED_ARM64_PLATFORMS: "linux macos" + CBCI_DEFAULT_LINUX_X86_64_PLATFORM: "ubuntu-22.04" + CBCI_DEFAULT_LINUX_ARM64_PLATFORM: "ubuntu-22.04-arm" + CBCI_DEFAULT_MACOS_X86_64_PLATFORM: "macos-13" + CBCI_DEFAULT_MACOS_ARM64_PLATFORM: "macos-14" + CBCI_DEFAULT_WINDOWS_PLATFORM: "windows-2022" + CBCI_DEFAULT_LINUX_CONTAINER: "slim-bookworm" + CBCI_DEFAULT_ALPINE_CONTAINER: "alpine" + CBCI_CBDINO_VERSION: "v0.0.80" + CI_SCRIPTS_URL: "https://raw.githubusercontent.com/couchbaselabs/sdkbuild-jenkinsfiles/master/python/ci_scripts_v1" + +jobs: + default-linux-runner: + runs-on: ubuntu-latest + outputs: + runner: ${{ steps.set_runner.outputs.runner }} + steps: + - id: set_runner + run: echo "runner=${{ env.CBCI_DEFAULT_LINUX_X86_64_PLATFORM }}" >> $GITHUB_OUTPUT + + ci-scripts: + needs: default-linux-runner + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + steps: + - name: Download CI Scripts + run: | + mkdir ci_scripts + cd ci_scripts + curl -o gha.sh ${CI_SCRIPTS_URL}/gha.sh + curl -o pygha.py ${CI_SCRIPTS_URL}/pygha.py + ls -alh + - name: Upload CI scripts + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: ci_scripts_verify_${{ inputs.packaging_index }} + path: | + ci_scripts/ + + validate-input: + needs: [ci-scripts, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + env: + CBCI_VERSION: ${{ inputs.version }} + CBCI_SHA: ${{ inputs.sha }} + CBCI_PACKAGING_INDEX: ${{ inputs.packaging_index }} + CBCI_CONFIG: ${{ inputs.config }} + steps: + - name: Download CI scripts + uses: actions/download-artifact@v4 + with: + name: ci_scripts_verify_${{ inputs.packaging_index }} + path: ci_scripts + - name: Verify Scripts + run: | + ls -alh ci_scripts + chmod +x ci_scripts/gha.sh + ls -alh ci_scripts + - name: Display workflow info + run: | + ./ci_scripts/gha.sh display_info + - name: Validate workflow info + run: | + ./ci_scripts/gha.sh validate_input ${{ github.workflow }} + + setup: + needs: [validate-input, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + env: + CBCI_CONFIG: ${{ inputs.config }} + CBCI_PACKAGING_INDEX: ${{ inputs.packaging_index }} + outputs: + stage_matrices: ${{ steps.build_matrices.outputs.stage_matrices }} + steps: + - name: Download CI scripts + uses: actions/download-artifact@v4 + with: + name: ci_scripts_verify_${{ inputs.packaging_index }} + path: ci_scripts + - name: Enable CI Scripts + run: | + chmod +x ci_scripts/gha.sh + - name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CBCI_DEFAULT_PYTHON }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Build stage matrices + id: build_matrices + run: | + exit_code=0 + STAGE_MATRICES=$(./ci_scripts/gha.sh get_stage_matrices) || exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Failed to obtain stage matrices." + exit 1 + fi + stage_matrices_json=$(jq -cn --argjson matrices "$STAGE_MATRICES" '$matrices') + echo "STAGE_MATRICES_JSON=$stage_matrices_json" + echo "stage_matrices=$stage_matrices_json" >> "$GITHUB_OUTPUT" + + confirm-matrices: + needs: [setup, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + steps: + - name: Alpine Test Unit Stage + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_alpine }} + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.alpine) }}" + - name: Linux Test Unit Stage + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_linux }} + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.linux) }}" + - name: Macos Test Unit Stage + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_macos }} + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.macos) }}" + - name: Windows Test Unit Stage + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_windows }} + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.windows) }}" + - name: Unit Test Commands + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest) }}" + - name: Integration Test Commands + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest) }}" + - name: Linux cbdino Stage + if: >- + ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.has_linux_cbdino + && !fromJson(needs.setup.outputs.stage_matrices).test_integration.skip_cbdino }} + run: | + echo cbdino config: + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_integration.cbdino_config) }}" + echo cbdino linux: + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_integration.linux_cbdino) }}" + - name: Linux Integration Stage + if: >- + ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.has_linux + && !fromJson(needs.setup.outputs.stage_matrices).test_integration.skip_integration }} + run: | + echo "${{ toJson(fromJson(needs.setup.outputs.stage_matrices).test_integration.test_config) }}" + + test-setup: + needs: [confirm-matrices, default-linux-runner] + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + env: + CBCI_CONFIG: ${{ inputs.config }} + steps: + - name: Checkout (with SHA) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + - name: Download CI scripts + uses: actions/download-artifact@v4 + with: + name: ci_scripts_verify_${{ inputs.packaging_index }} + path: ci_scripts + - name: Enable CI Scripts + run: | + chmod +x ci_scripts/gha.sh + - name: Setup Python ${{ env.CBCI_DEFAULT_PYTHON }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.CBCI_DEFAULT_PYTHON }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Build test setup + run: | + ./ci_scripts/gha.sh build_test_setup + - name: Confirm test setup + run: | + echo "pycbac_test directory contents:" + ls -alh pycbac_test + echo "pycbac_test/acb/tests contents:" + ls -alh pycbac_test/acb/tests + echo "pycbac_test/cb/tests contents:" + ls -alh pycbac_test/cb/tests + echo "pycbac_test/tests contents:" + ls -alh pycbac_test/tests + echo "pycbac_test/conftest.py contents:" + cat pycbac_test/conftest.py + echo "pycbac_test/requirements-test.txt contents:" + cat pycbac_test/requirements-test.txt + echo "pycbac_test/pytest.ini contents:" + cat pycbac_test/pytest.ini + echo "pycbac_test/tests/test_config.ini contents:" + cat pycbac_test/tests/test_config.ini + - name: Upload test setup + uses: actions/upload-artifact@v4 + with: + retention-days: 1 + name: pycbac-test-setup-verify-${{ inputs.packaging_index }} + path: | + pycbac_test/ + + alpine-unit-tests: + needs: [setup, test-setup, default-linux-runner] + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_alpine }} + runs-on: ${{ needs.default-linux-runner.outputs.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.alpine }} + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbac-test-setup-verify-${{ inputs.packaging_index }} + path: pycbac + - name: Cache Docker image + uses: actions/cache@v4 + with: + path: /tmp/docker_python_image.tar # Path to store the image archive + key: docker-image-python-${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} + - name: Pull and save image if not cached + run: | + if [ ! -f /tmp/docker_python_image.tar ]; then + docker pull python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} + docker save python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} -o /tmp/docker_python_image.tar + fi + - name: Load image if cached + run: | + if [ -f /tmp/docker_python_image.tar ]; then + docker load -i /tmp/docker_python_image.tar + fi + - name: Run unit tests in docker via pip install (${{ inputs.packaging_index }}) + uses: addnab/docker-run-action@v3 + with: + image: python:${{ matrix.python-version }}-${{ env.CBCI_DEFAULT_ALPINE_CONTAINER }} + options: >- + --platform linux/amd64 + -v ${{ github.workspace }}/pycbac:/pycbac + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + ls -alh + python -m pip install -r requirements-test.txt + INSTALL_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.install_cmd }}" + SDK_VERSION="${{ inputs.version }}" + echo "INSTALL_CMD=$INSTALL_CMD" + PIP_CMD="python -m pip $INSTALL_CMD couchbase-analytics==$SDK_VERSION" + eval "$PIP_CMD" + python -m pip list + TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} + if [ "$TEST_ACOUCHBASE_API" = "true" ]; then + PYTEST_ACB_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }}" + PYTEST_ACB_OPTS="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }}" + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} + if [ "$TEST_COUCHBASE_API" = "true" ]; then + PYTEST_CB_CMD="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }}" + PYTEST_CB_OPTS="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }}" + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + + linux-unit-tests: + needs: [setup, test-setup] + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_linux }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.linux }} + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbac-test-setup-verify-${{ inputs.packaging_index }} + path: pycbac + - name: Run unit tests via pip install (${{ inputs.packaging_index }}) + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} + INSTALL_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.install_cmd }} + SDK_VERSION: ${{ inputs.version }} + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + ls -alh + python -m pip install -r requirements-test.txt + echo "INSTALL_CMD=$INSTALL_CMD" + PIP_CMD="python -m pip $INSTALL_CMD couchbase-analytics==$SDK_VERSION" + eval "$PIP_CMD" + python -m pip list + TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} + if [ "$TEST_ACOUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} + if [ "$TEST_COUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + + macos-unit-tests: + needs: [setup, test-setup] + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_macos }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.macos }} + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbac-test-setup-verify-${{ inputs.packaging_index }} + path: pycbac + - name: Run unit tests via pip install (${{ inputs.packaging_index }}) + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} + INSTALL_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.install_cmd }} + SDK_VERSION: ${{ inputs.version }} + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + ls -alh + python -m pip install -r requirements-test.txt + echo "INSTALL_CMD=$INSTALL_CMD" + PIP_CMD="python -m pip $INSTALL_CMD couchbase-analytics==$SDK_VERSION" + eval "$PIP_CMD" + python -m pip list + TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }} + if [ "$TEST_ACOUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }} + if [ "$TEST_COUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + + windows-unit-tests: + needs: [setup, test-setup] + if: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.has_windows }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.windows }} + steps: + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbac-test-setup-verify-${{ inputs.packaging_index }} + path: pycbac + - name: Run unit tests via pip install (${{ inputs.packaging_index }}) + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.pytest.couchbase_opts }} + INSTALL_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.install_cmd }} + SDK_VERSION: ${{ inputs.version }} + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + dir + python -m pip install -r requirements-test.txt + echo "INSTALL_CMD=$env:INSTALL_CMD" + PIP_CMD="python -m pip $($env:INSTALL_CMD) couchbase-analytics==$($env:SDK_VERSION)" + python -m pip list + $TEST_ACOUCHBASE_API="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_acouchbase_api }}" + if ( $TEST_ACOUCHBASE_API -eq "true" ) { + $PYTEST_CMD="python -m $($env:PYTEST_ACB_CMD) $($env:PYTEST_ACB_OPTS)" + echo "Running acouchbase tests: $PYTEST_CMD" + iex $PYTEST_CMD + } + $TEST_COUCHBASE_API="${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.test_couchbase_api }}" + if ( $TEST_COUCHBASE_API = "true" ) { + $PYTEST_CMD="python -m $($env:PYTEST_CB_CMD) $($env:PYTEST_CB_OPTS)" + echo "Running couchbase tests: $PYTEST_CMD" + iex $PYTEST_CMD + } + + cbdino-integration-tests: + needs: [setup, test-setup] + if: >- + ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.has_linux_cbdino + && !fromJson(needs.setup.outputs.stage_matrices).test_integration.skip_cbdino }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.linux_cbdino }} + steps: + - name: Install cbdinocluster + run: | + mkdir -p "$HOME/bin" + CB_DINO_VERSION=${{ env.CBCI_CBDINO_VERSION }} + CB_DINO_TYPE="cbdinocluster-${{ matrix.arch == 'x86_64' && 'linux-amd64' || 'linux-arm64' }}" + wget -nv -O $HOME/bin/cbdinocluster https://github.com/couchbaselabs/cbdinocluster/releases/download/$CB_DINO_VERSION/$CB_DINO_TYPE + chmod +x $HOME/bin/cbdinocluster + echo "$HOME/bin" >> $GITHUB_PATH + - name: Install s3mock + run: | + docker pull adobe/s3mock + docker pull nginx + - name: Initialize cbdinocluster + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cbdinocluster -v init --auto + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Confirm Python version + run: python -c "import sys; print(sys.version)" + - name: Download CI scripts + uses: actions/download-artifact@v4 + with: + name: ci_scripts_verify_${{ inputs.packaging_index }} + path: ci_scripts + - name: Enable CI Scripts + run: | + chmod +x ci_scripts/gha.sh + - name: Download test setup + uses: actions/download-artifact@v4 + with: + name: pycbac-test-setup-verify-${{ inputs.packaging_index }} + path: pycbac + - name: Start couchbase cluster + run: | + cd pycbac + cat cluster_def.yaml + CBDC_ID=$(cbdinocluster -v alloc --def-file=cluster_def.yaml) + CBDC_CONNSTR=$(cbdinocluster -v connstr --analytics $CBDC_ID) + echo "CBDC_ID=$CBDC_ID" >> "$GITHUB_ENV" + echo "CBDC_CONNSTR=$CBDC_CONNSTR" >> "$GITHUB_ENV" + echo "CBDC_CONNSTR=$CBDC_CONNSTR" + cbdinocluster buckets load-sample $CBDC_ID travel-sample + - name: Update test_config.ini + env: + PYCBAC_USERNAME: 'Administrator' + PYCBAC_PASSWORD: 'password' + PYCBAC_FQDN: 'travel-sample.inventory.airline' + CBCONNSTR: ${{ env.CBDC_CONNSTR }} + run: | + ./ci_scripts/gha.sh build_test_config_ini pycbac/tests + - name: Run tests + timeout-minutes: 30 + env: + PYTEST_ACB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.acouchbase_cmd }} + PYTEST_ACB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.acouchbase_opts }} + PYTEST_CB_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.couchbase_cmd }} + PYTEST_CB_OPTS: ${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.pytest.couchbase_opts }} + INSTALL_CMD: ${{ fromJson(needs.setup.outputs.stage_matrices).test_unit.install_cmd }} + SDK_VERSION: ${{ inputs.version }} + run: | + python -m pip install --upgrade pip setuptools wheel + cd pycbac + ls -alh + cat tests/test_config.ini + python -m pip install -r requirements-test.txt + echo "INSTALL_CMD=$INSTALL_CMD" + PIP_CMD="python -m pip $INSTALL_CMD couchbase-analytics==$SDK_VERSION" + eval "$PIP_CMD" + python -m pip list + TEST_ACOUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.test_acouchbase_api }} + if [ "$TEST_ACOUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_ACB_CMD $PYTEST_ACB_OPTS" + echo "Running acouchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + TEST_COUCHBASE_API=${{ fromJson(needs.setup.outputs.stage_matrices).test_integration.test_couchbase_api }} + if [ "$TEST_COUCHBASE_API" = "true" ]; then + PYTEST_CMD="python -m $PYTEST_CB_CMD $PYTEST_CB_OPTS" + echo "Running couchbase tests: $PYTEST_CMD" + eval "$PYTEST_CMD" + fi + - name: Cleanup cbdino cluster + run: | + cbdinocluster rm ${{ env.CBDC_ID }}