diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9f20ee93f..b0be4a542 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,8 +34,6 @@ jobs: arch: x86_64 - os: ubuntu-24.04-arm arch: aarch64 - - os: ubuntu-24.04 - arch: armv7l - os: ubuntu-24.04 arch: x86_64 - os: windows-latest @@ -47,11 +45,6 @@ jobs: - uses: actions/setup-python@v6 with: python-version: "3.14" - - name: Set up QEMU - if: matrix.arch == 'armv7l' - uses: docker/setup-qemu-action@v3 - with: - platforms: arm - name: Set Minimum MacOS Target if: runner.os == 'macOS' run: | @@ -73,7 +66,6 @@ jobs: CIBW_BUILD: "cp311* cp314t*" CIBW_TEST_COMMAND: mv {project}/av {project}/av.disabled && python -m pytest {package}/tests && mv {project}/av.disabled {project}/av CIBW_TEST_REQUIRES: pytest numpy - CIBW_TEST_SKIP: "*_armv7l" run: | pip install cibuildwheel delvewheel cibuildwheel --output-dir dist @@ -84,9 +76,42 @@ jobs: name: dist-${{ matrix.os }}-${{ matrix.arch }} path: dist/ + # armv7l (32-bit ARM) is cross-compiled with zig on a native x86_64 runner + # instead of emulated with QEMU. cibuildwheel cannot cross-compile, so this + # target uses its own driver (scripts/build-armv7l-cross.sh). + package-wheel-armv7l: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + - name: Set up host Python 3.11 + uses: actions/setup-python@v6 + with: + python-version: "3.11" + - name: Set up host Python 3.14t + uses: actions/setup-python@v6 + with: + python-version: "3.14t" + - name: Install build tooling + run: | + for py in python3.11 python3.14t; do + $py -m pip install -U "cython>=3.1.0,<4" "setuptools>=77.0" wheel + done + python3.11 -m pip install -U ziglang auditwheel + - name: Cross-compile armv7l wheels with zig + env: + HOST_PY311: python3.11 + HOST_PY314T: python3.14t + TOOLS_PY: python3.11 + run: bash scripts/build-armv7l-cross.sh + - name: Upload wheels + uses: actions/upload-artifact@v6 + with: + name: dist-armv7l + path: dist/ + publish: runs-on: ubuntu-latest - needs: [package-source, package-wheel] + needs: [package-source, package-wheel, package-wheel-armv7l] steps: - uses: actions/checkout@v6 - uses: actions/download-artifact@v7 diff --git a/scripts/auditwheel-cross.py b/scripts/auditwheel-cross.py new file mode 100644 index 000000000..13c529dc7 --- /dev/null +++ b/scripts/auditwheel-cross.py @@ -0,0 +1,28 @@ +"""Run ``auditwheel`` for a foreign architecture/libc from an x86_64 host. + +auditwheel builds its ``--plat`` choices from the host's detected architecture +and libc, so on an x86_64 runner it rejects e.g. ``manylinux_2_31_armv7l``. The +repair logic itself re-derives the architecture and libc from the wheel and works +cross-arch, so we only need to override the host detection used to build the CLI +choice list. + +Usage:: + + python auditwheel-cross.py +""" + +import sys + +from auditwheel.architecture import Architecture +from auditwheel.libc import Libc + +_arch = Architecture(sys.argv[1]) +_libc = {"glibc": Libc.GLIBC, "musl": Libc.MUSL}[sys.argv[2]] +del sys.argv[1:3] + +Architecture.detect = staticmethod(lambda *, bits=None: _arch) +Libc.detect = staticmethod(lambda: _libc) + +from auditwheel.main import main # noqa: E402 (import after patching detection) + +sys.exit(main()) diff --git a/scripts/build-armv7l-cross.sh b/scripts/build-armv7l-cross.sh new file mode 100755 index 000000000..69eb3bfea --- /dev/null +++ b/scripts/build-armv7l-cross.sh @@ -0,0 +1,130 @@ +#! /usr/bin/env bash +# +# Cross-compile PyAV's armv7l (32-bit ARM) wheels with zig on an x86_64 host, +# without QEMU. Only PyAV's Cython->C extensions are compiled; FFmpeg is fetched +# prebuilt for armv7l. zig is the cross compiler, and CPython's cross-build env +# vars (_PYTHON_HOST_PLATFORM / _PYTHON_SYSCONFIGDATA_NAME) make sysconfig report +# armv7l without running any armv7l code. The target Python trees + system libs +# are copied out of the manylinux armv7l image with `docker cp` (no execution). +# +# Builds two manylinux (glibc) wheels: cp311-abi3 (covers 3.11-3.13) and cp314t. +# musllinux armv7l is skipped: the musl FFmpeg needs system libs (libdrm, +# libxcb*, libbz2) the musllinux image doesn't ship for auditwheel to bundle. + +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +root="$(cd "$here/.." && pwd)" +cd "$root" + +# Host interpreters that drive each build (must match the wheel's Python: 3.11 +# emits cp311-abi3, 3.14t emits cp314t). zig/auditwheel run from TOOLS_PY. +HOST_PY311="${HOST_PY311:-python3.11}" +HOST_PY314T="${HOST_PY314T:-python3.14t}" +TOOLS_PY="${TOOLS_PY:-python3}" + +DIST_DIR="${DIST_DIR:-$root/dist}" +WORK_DIR="${WORK_DIR:-$root/build/armv7l-cross}" +IMAGE="quay.io/pypa/manylinux_2_31_armv7l" +TRIPLE="arm-linux-gnueabihf.2.31" # pin glibc to the manylinux_2_31 policy +PLAT="manylinux_2_31_armv7l" +FFMPEG_CONFIG="scripts/ffmpeg-latest.json" + +PY_DIR="$WORK_DIR/py" +BIN_DIR="$WORK_DIR/bin" +mkdir -p "$DIST_DIR" "$BIN_DIR" "$PY_DIR" + +# --- zig compiler wrappers ------------------------------------------------ +# setuptools invokes $CC / $LDSHARED as plain commands. Resolve the zig binary +# once and call it directly (not "python -m ziglang"): the build sets PYTHONPATH +# to the target stdlib, which would crash a per-compile Python. The wrappers also +# drop a few gcc-only flags clang (zig) rejects. +ZIG_BIN="$("$TOOLS_PY" -c 'import os, ziglang; print(os.path.join(os.path.dirname(ziglang.__file__), "zig"))')" + +write_cc() { + cat >"$1" <"$BIN_DIR/zig-ar" +chmod +x "$BIN_DIR/zig-ar" + +# --- copy the target Python trees + system libs out of the image ---------- +extract_image() { + if [[ -d "$PY_DIR/_internal" ]]; then + return # already extracted + fi + echo "Pulling $IMAGE" + docker pull --platform linux/arm/v7 "$IMAGE" + local cid + cid="$(docker create --platform linux/arm/v7 "$IMAGE")" + # /opt/python/ are symlinks into /opt/_internal/; copy both. Also + # grab the armhf system libs (libxcb, ...) FFmpeg needs so auditwheel can + # bundle them. docker cp reads the filesystem; it runs no armv7l code. + docker cp "$cid:/opt/python" "$PY_DIR" + docker cp "$cid:/opt/_internal" "$PY_DIR" + docker cp "$cid:/usr/lib/arm-linux-gnueabihf" "$PY_DIR/syslib" + docker rm -f "$cid" >/dev/null +} + +# --- build + repair one wheel --------------------------------------------- +# args: +build_one() { + local host_py="$1" py_glob="$2" + + # Resolve /opt/python/ symlink to its real tree under /opt/_internal. + local link + link="$(find "$PY_DIR/python" -maxdepth 1 -name "$py_glob" | head -1)" + [[ -n "$link" ]] || { echo "no python matching '$py_glob':" >&2; ls "$PY_DIR/python" >&2; exit 1; } + local pytree="$PY_DIR/_internal/$(basename "$(readlink "$link")")" + + local inc=( "$pytree"/include/python3.* ) + local scd=( "$pytree"/lib/python3.*/_sysconfigdata_*.py ) + echo "=== building for $(basename "$pytree") (host $host_py) ===" + + # FFmpeg's pkg-config files bake in prefix=/tmp/vendor, so extract there. + rm -rf /tmp/vendor + PYAV_VENDOR_PLATFORM=manylinux-armv7l "$TOOLS_PY" scripts/fetch-vendor.py \ + --config-file "$FFMPEG_CONFIG" /tmp/vendor + + local raw="$WORK_DIR/raw/$(basename "$pytree")" + rm -rf "$raw"; mkdir -p "$raw" + + # The _PYTHON_* vars make sysconfig report armv7l; zig does the compiling. + env \ + CC="$BIN_DIR/zig-cc" CXX="$BIN_DIR/zig-cxx" AR="$BIN_DIR/zig-ar" \ + LDSHARED="$BIN_DIR/zig-cc -shared" \ + _PYTHON_HOST_PLATFORM=linux-armv7l \ + _PYTHON_SYSCONFIGDATA_NAME="$(basename "${scd[0]}" .py)" \ + PYTHONPATH="$(dirname "${scd[0]}")" \ + CFLAGS="-I${inc[0]} -Wno-error=incompatible-pointer-types" \ + PKG_CONFIG_PATH=/tmp/vendor/lib/pkgconfig \ + LD_LIBRARY_PATH=/tmp/vendor/lib \ + "$host_py" -m pip wheel . --no-build-isolation --no-deps -w "$raw" + + # Bundle FFmpeg + its armhf system deps and stamp the platform tag. The shim + # lets auditwheel accept the armv7l --plat from an x86_64 host; --ldpaths + # points it at the target's libs instead of the host search path. + "$TOOLS_PY" scripts/auditwheel-cross.py armv7l glibc repair \ + --ldpaths "/tmp/vendor/lib:$PY_DIR/syslib" \ + --plat "$PLAT" -w "$DIST_DIR" "$raw"/*.whl +} + +extract_image +build_one "$HOST_PY311" "cp311-cp311" +build_one "$HOST_PY314T" "cp314*-cp314t" + +echo +echo "Built armv7l wheels:" +ls -1 "$DIST_DIR"/*armv7l*.whl diff --git a/scripts/fetch-vendor.py b/scripts/fetch-vendor.py index 7161a0023..b046c1884 100644 --- a/scripts/fetch-vendor.py +++ b/scripts/fetch-vendor.py @@ -8,6 +8,12 @@ def get_platform(): + # Allow forcing the target platform so we can fetch e.g. an armv7l build + # while running on an x86_64 host (cross-compilation). + forced = os.environ.get("PYAV_VENDOR_PLATFORM") + if forced: + return forced + system = platform.system() machine = platform.machine().lower() is_arm64 = machine in {"arm64", "aarch64"}