From 3204768de7d40d47a8afdbe8f3b23b174e73fbb1 Mon Sep 17 00:00:00 2001 From: Ludovic Henry Date: Wed, 19 Aug 2026 21:25:22 +0200 Subject: [PATCH] workflows: add build-protobuf.yml for riscv64 manywheel builds protobuf's setup.py cannot build from the git checkout (upstream's python README is explicit about this); the self-contained sdist that bundles the upb/utf8_range C sources and generated *_pb2.py files is assembled by Bazel via //python/dist:source_wheel. Mirror build-cffi.yml's sdist -> bdist split: - python_sdist: Bazel-build the architecture-independent sdist once on x86. - linux: build the riscv64 bdist from that sdist with cibuildwheel. The bundled C sources compile into google._upb._message with no protoc, Bazel, or external native dependency, so no CIBW_BEFORE_BUILD is needed. The test asserts the upb backend is active and round-trips a well-known type. - publish: hand off to the shared publish-wheels action. The workflow input is the git tag (e.g. 35.1); the Python package version (7.35.1) is derived from the built sdist filename. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build-protobuf.yml | 260 +++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100644 .github/workflows/build-protobuf.yml diff --git a/.github/workflows/build-protobuf.yml b/.github/workflows/build-protobuf.yml new file mode 100644 index 00000000..3a2532d5 --- /dev/null +++ b/.github/workflows/build-protobuf.yml @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +name: Build protobuf wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'protobuf version to build (e.g. 7.35.1)' + required: true + default: '7.35.1' + pull_request: + paths: + - '.github/workflows/build-protobuf.yml' + +concurrency: + group: ${{ github.workflow }}-${{ inputs.version || '7.35.1' }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + PROTOBUF_VERSION: ${{ inputs.version || '7.35.1' }} + UV_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + UV_INDEX_STRATEGY: unsafe-best-match + UV_ONLY_BINARY: ':all:' + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + +jobs: + python_sdist: + # protobuf's setup.py CANNOT build from the git checkout - its python + # README is explicit: "You cannot build from setup.py using the GitHub repo + # or the GitHub source tarball." The self-contained sdist (which bundles the + # upb and utf8_range C sources plus the generated *_pb2.py / *.upb.c files) + # is assembled by Bazel via //python/dist:source_wheel. That sdist is + # architecture-independent ("always the same and does not depend on a local + # version of Python"), so we build it once here on x86 - building protoc + # with Bazel on riscv64 is not a path worth maintaining. The riscv64 bdist + # is then built from this sdist in the `linux` job below, mirroring + # build-cffi.yml's sdist -> bdist split. + # + # Note: the package version (7.NN.M) and git tag (vNN.M) differ - a "derive + # git tag" step strips the leading "7." to get the tag used for checkout. + runs-on: ubuntu-latest + outputs: + sdist_artifact_name: ${{ steps.build_sdist.outputs.sdist_artifact_name }} + testwheel_artifact_name: ${{ steps.build_sdist.outputs.testwheel_artifact_name }} + package_version: ${{ steps.build_sdist.outputs.package_version }} + env: + # No .bazelversion is committed at the release tags, so bazelisk would + # otherwise track "latest". Pin to the version protobuf's own CI uses, + # which satisfies its MODULE.bazel (bazel_compatibility = ">=8.0.0"). + USE_BAZEL_VERSION: '8.7.0' + steps: + - name: derive git tag + # protobuf package versions are 7.NN.M but git tags are vNN.M (no major). + # Strip the leading "7." to get the tag, then export it for downstream steps. + id: tag + run: | + tag="${PROTOBUF_VERSION#7.}" + echo "value=${tag}" >> "$GITHUB_OUTPUT" + + - name: checkout protobuf v${{ steps.tag.outputs.value }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + repository: protocolbuffers/protobuf + ref: v${{ steps.tag.outputs.value }} + # protobuf uses bzlmod, not git submodules, so a plain checkout is + # enough (unlike build-cffi.yml, which needs submodules: true). + persist-credentials: false + + - name: setup python + # Bazel's system_python extension resolves the host interpreter from + # PATH; //python/dist:source_wheel is target_compatible_with a found + # system python, so one must be present for the target to build. + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: install bazelisk + # Don't rely on Bazel being pre-installed on the runner image. + run: | + set -eux + curl -fsSL \ + https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-amd64 \ + -o "${RUNNER_TEMP}/bazel" + chmod +x "${RUNNER_TEMP}/bazel" + echo "${RUNNER_TEMP}" >> "$GITHUB_PATH" + + - name: build sdist + id: build_sdist + run: | + set -eux + + # Bazel assembles the self-contained Python source distribution + # (compiles protoc to generate the bootstrap *_pb2.py, bundles upb). + # //python/dist:test_wheel produces protobuftests--py3-none-any.whl, + # an architecture-independent wheel bundling the *_test.py suite plus + # the generated test protos/testdata. The riscv64 `linux` job installs + # it to run protobuf's real test suite (see test_upb.yml upstream), so + # we build it here once on x86 alongside the sdist. + bazel build //python/dist:source_wheel //python/dist:test_wheel + + rm -rf dist/ + mkdir -p dist + cp bazel-bin/python/dist/protobuf-*.tar.gz dist/ + cp "$(find bazel-bin/python/dist -name 'protobuftests-*.whl' | head -1)" dist/ + + sdist_name="$(basename "$(echo dist/protobuf-*.tar.gz)")" + testwheel_name="$(basename "$(echo dist/protobuftests-*.whl)")" + package_version="$(echo "${sdist_name}" | sed -En 's/protobuf-(.+)\.tar\.gz/\1/p')" + { + echo "sdist_artifact_name=${sdist_name}" + echo "testwheel_artifact_name=${testwheel_name}" + echo "package_version=${package_version}" + } >> "$GITHUB_OUTPUT" + + - name: upload sdist artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.build_sdist.outputs.sdist_artifact_name }} + path: dist/${{ steps.build_sdist.outputs.sdist_artifact_name }} + if-no-files-found: error + # always upload the sdist artifact- the wheel build job requires it + + - name: upload test wheel artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.build_sdist.outputs.testwheel_artifact_name }} + path: dist/${{ steps.build_sdist.outputs.testwheel_artifact_name }} + if-no-files-found: error + # the riscv64 build job installs this to run the test suite + + linux: + # Unlike upstream (which builds a large cross-compiled matrix via Bazel), we + # build the riscv64 bdist from the sdist with cibuildwheel. protobuf's + # setup.py compiles the bundled upb/utf8_range C sources into the + # google._upb._message extension - no protoc or Bazel is needed at this + # stage, and no external native dependency (so, no CIBW_BEFORE_BUILD). + needs: [python_sdist] + name: Build protobuf ${{ needs.python_sdist.outputs.package_version }} ${{ matrix.python }}-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + strategy: + fail-fast: false + matrix: + python: ["cp312", "cp313", "cp314", "cp314t"] + + steps: + - name: fetch sdist artifact + id: fetch_sdist + uses: actions/download-artifact@v8 + with: + name: ${{ needs.python_sdist.outputs.sdist_artifact_name }} + path: sdist + + - name: fetch test wheel artifact + id: fetch_testwheel + uses: actions/download-artifact@v8 + with: + name: ${{ needs.python_sdist.outputs.testwheel_artifact_name }} + path: testwheel + + - name: setup uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + python-version: '3.12' + activate-environment: true + enable-cache: false + + - name: build/test wheels + id: build + env: + CIBW_BUILD: ${{ matrix.python }}-manylinux_riscv64 + CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} + # Run protobuf's real Python test suite against the freshly built wheel, + # the same way upstream's test_upb.yml does: install the protobuftests + # wheel, then discover and run every *_test.py via unittest (excluding + # the two C++/pybind11-backend modules, which don't apply to upb). The + # runner script and test wheel are staged under {package}/_testwheel so + # cibuildwheel copies them into the manylinux container. + CIBW_TEST_REQUIRES: numpy<=2.3.4 absl-py==2.* setuptools<=78.1.1 + # cibuildwheel expands {package} to the package dir we pass below + # (./protobuf, copied to /project/protobuf in the container), which is + # where we stage the runner and test wheel. ({project} is its parent.) + CIBW_TEST_COMMAND: bash {package}/_testwheel/run_tests.sh + # numpy has no riscv64 wheel on public PyPI; pull it (and anything else + # missing) from our registry inside the test venv. Same mechanism as + # build-onnx.yml. + CIBW_ENVIRONMENT_PASS_LINUX: PIP_EXTRA_INDEX_URL + PIP_EXTRA_INDEX_URL: https://pypi.riseproject.dev/simple/ + run: | + set -eux + + mkdir protobuf + + tar zxf sdist/protobuf-*.tar.gz --strip-components=1 -C protobuf + + # Stage the arch-independent protobuftests wheel and the test runner + # inside the package dir so both are available at {package} in-container. + # The runner mirrors upstream test_upb.yml: verify the upb extension is + # importable, install the test wheel, then run every discovered *_test.py + # module (minus the C++/pybind11-backend ones) and fail if any fails. + mkdir protobuf/_testwheel + cp testwheel/protobuftests-*.whl protobuf/_testwheel/ + cat > protobuf/_testwheel/run_tests.sh <<'EOF' + #!/usr/bin/env bash + set -eu + here="$(cd "$(dirname "$0")" && pwd)" + + # Sanity-check the upb C extension is present and wired up before testing. + python -c "from google._upb import _message; assert 'google._upb._message.MessageMeta' in str(_message.MessageMeta)" + + pip install --no-deps "$here"/protobuftests-*.whl + + tests="$(pip show -f protobuftests \ + | grep _test.py \ + | grep --invert-match -e _pybind11_test.py -e proto_api_test.py \ + | sed 's,[/\\],.,g' \ + | sed -E 's,.py$,,g')" + + rc=0 + for t in $tests; do + echo "::group::$t" + python -m unittest -v "$t" || rc=1 + echo "::endgroup::" + done + exit $rc + EOF + + uv pip install --upgrade cibuildwheel + + python -m cibuildwheel --output-dir dist ./protobuf + + echo "artifact_name=$(ls ./dist/)" >> "$GITHUB_OUTPUT" + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: protobuf-${{ needs.python_sdist.outputs.package_version }}-${{ matrix.python }}-manylinux_riscv64 + path: ./dist/*.whl + if-no-files-found: error + + publish: + name: Publish protobuf ${{ needs.python_sdist.outputs.package_version }} to GitLab + needs: [python_sdist, linux] + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Publish wheels and open docs PR + uses: riseproject-dev/python-wheels/actions/publish-wheels@main + with: + artifact-pattern: protobuf-${{ needs.python_sdist.outputs.package_version }}-*-manylinux_riscv64 + gitlab-username: ${{ vars.GITLAB_DEPLOY_USER }} + gitlab-token: ${{ secrets.GITLAB_DEPLOY_TOKEN }} + gitlab-project-id: ${{ vars.GITLAB_PROJECT_ID }} + gh-token: ${{ secrets.GITHUB_TOKEN }}