diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d0e4f5309..a269f5a1d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -140,7 +140,7 @@ jobs: - name: Avalonia headless tests run: dotnet test tests/WebScene.Backend.Avalonia.Tests/WebScene.Backend.Avalonia.Tests.csproj -c Release --no-build - name: WPT manifest integrity - run: dotnet run --project tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj -c Release --no-build -- --selection all --list + run: dotnet run --project tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj -c Release -f net10.0 --no-build -- --selection all --list - name: Native C++ portability build (without V8) run: >- cmake diff --git a/.github/workflows/native-runtime-packages.yml b/.github/workflows/native-runtime-packages.yml index 1b67d0764..d580215ab 100644 --- a/.github/workflows/native-runtime-packages.yml +++ b/.github/workflows/native-runtime-packages.yml @@ -8,6 +8,16 @@ on: required: true default: false type: boolean + consumer_recheck_run_id: + description: Existing run whose verified release artifact should be rechecked + required: false + default: '' + type: string + consumer_recheck_version: + description: Package version in the existing release artifact + required: false + default: '' + type: string push: branches: - main @@ -24,7 +34,10 @@ on: - 'experiments/WebScene.NativeEngine.Probe/**' - 'packaging/WebScene.NativeEngine.Runtime/**' - 'scripts/build-native-engine-runtime.sh' - - 'scripts/build-native-engine-runtime-linux-container.sh' + - 'scripts/build-linux-native-runtime.sh' + - 'scripts/linux-glibc-toolchain.cmake' + - 'scripts/verify-linux-native-abi.py' + - 'scripts/verify-native-payload-reproducibility.py' - 'scripts/build-native-engine-runtime.ps1' - 'scripts/V8WindowsEnvironment.psm1' - 'scripts/pack-packages.sh' @@ -57,6 +70,7 @@ permissions: jobs: metadata: + if: inputs.consumer_recheck_run_id == '' runs-on: [self-hosted, Linux, X64] outputs: package-version: ${{ steps.version.outputs.package-version }} @@ -77,6 +91,9 @@ jobs: scripts/tests/test_native_runtime_workflow_policy.py \ scripts/tests/test_prepare_dotnet_install.py \ scripts/tests/test_prepare_runner_disk.py \ + scripts/tests/test_linux_build_policy.py \ + scripts/tests/test_verify_linux_native_abi.py \ + scripts/tests/test_verify_native_payload_reproducibility.py \ scripts/tests/test_verify_cross_rid_compatibility.py \ scripts/tests/test_v8_bootstrap_literals.py - name: Repair incomplete .NET install @@ -189,37 +206,139 @@ jobs: artifacts/nuget-packages/packages.json if-no-files-found: error + linux-builder: + name: Build immutable Linux cross-builder + needs: metadata + runs-on: [self-hosted, Linux, X64] + permissions: + contents: read + packages: write + outputs: + image: ${{ steps.reference.outputs.image }} + arm64-finalizer-image: ${{ steps.finalizer-reference.outputs.image }} + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - uses: docker/setup-buildx-action@v3 + - name: Log in to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - id: build + name: Build pinned Linux cross-builder + uses: docker/build-push-action@v6 + with: + context: packaging/WebScene.NativeEngine.Runtime + file: packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc + platforms: linux/amd64 + push: ${{ github.event_name != 'pull_request' }} + tags: ghcr.io/scenetech/webscene-linux-builder:webscene-linux-glibc-v1 + - id: reference + name: Resolve immutable builder reference + if: github.event_name != 'pull_request' + shell: bash + run: echo "image=ghcr.io/scenetech/webscene-linux-builder@${{ steps.build.outputs.digest }}" >> "$GITHUB_OUTPUT" + - id: finalizer-build + name: Build pinned Linux ARM64 finalizer + uses: docker/build-push-action@v6 + with: + context: packaging/WebScene.NativeEngine.Runtime + file: packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-arm64-finalizer + platforms: linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + load: ${{ github.event_name == 'pull_request' }} + tags: ghcr.io/scenetech/webscene-linux-arm64-finalizer:webscene-linux-glibc-v1 + - id: finalizer-reference + name: Resolve immutable ARM64 finalizer reference + shell: bash + run: | + if [[ '${{ github.event_name }}' == pull_request ]]; then + echo "image=ghcr.io/scenetech/webscene-linux-arm64-finalizer:webscene-linux-glibc-v1" >> "$GITHUB_OUTPUT" + else + echo "image=ghcr.io/scenetech/webscene-linux-arm64-finalizer@${{ steps.finalizer-build.outputs.digest }}" >> "$GITHUB_OUTPUT" + fi + native: name: Build ${{ matrix.rid }} - needs: metadata + needs: [metadata, linux-builder] + permissions: + actions: read + contents: read + packages: read strategy: fail-fast: false matrix: include: - os: [self-hosted, macOS, ARM64] rid: osx-arm64 + dotnet_architecture: arm64 cpu: arm64 monolith: libv8_monolith.a script: unix v8_revision: 15.3.10 partition_alloc: true v8_configuration: ReleasePartitionAlloc - v8_cache_generation: v9-v8-15.3.10-pa-no-process-shim-macos-inspector + v8_cache_generation: v10-v8-15.3.10-pa-system-libcxx-macos-inspector + v8_cache_extra_paths: '' + v8_cache_script: scripts/build-native-engine-runtime.sh + v8_cache_patch: packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocMacVisibilityPatch.txt + - os: [self-hosted, macOS, ARM64] + rid: osx-x64 + dotnet_architecture: x64 + cpu: x64 + monolith: libv8_monolith.a + script: unix + v8_revision: 15.3.10 + partition_alloc: true + v8_configuration: ReleasePartitionAlloc + v8_cache_generation: v10-v8-15.3.10-pa-system-libcxx-macos-inspector + v8_cache_extra_paths: '' v8_cache_script: scripts/build-native-engine-runtime.sh v8_cache_patch: packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocMacVisibilityPatch.txt - os: [self-hosted, Linux, X64] rid: linux-x64 + dotnet_architecture: x64 cpu: x64 monolith: libv8_monolith.a script: unix v8_revision: 15.3.10 partition_alloc: true v8_configuration: ReleasePartitionAlloc - v8_cache_generation: v12-v8-15.3.10-pa-no-process-shim-ubuntu22-gcc12-lld-no-crel-shared-inspector + v8_cache_generation: v2-v8-15.3.10-glibc227-cross-x64 + v8_cache_extra_paths: | + artifacts/native-engine-v8/linux-*/v8/buildtools/third_party/libc++ + artifacts/native-engine-v8/linux-*/v8/out/*/*/obj/buildtools/third_party/libc++abi/libc++abi.a + artifacts/native-engine-v8/linux-*/v8/third_party/libc++/src/include + artifacts/native-engine-v8/linux-*/v8/third_party/libc++abi/src/include + artifacts/native-engine-v8/linux-*/v8/third_party/llvm-build/Release+Asserts + v8_cache_script: scripts/build-native-engine-runtime.sh + v8_cache_patch: packaging/WebScene.NativeEngine.Runtime/patches/V8BuildNoCrelPatch.txt + - os: [self-hosted, Linux, X64] + rid: linux-arm64 + dotnet_architecture: x64 + cpu: arm64 + monolith: libv8_monolith.a + script: unix + v8_revision: 15.3.10 + partition_alloc: true + v8_configuration: ReleasePartitionAlloc + v8_cache_generation: v3-v8-15.3.10-glibc227-cross-arm64 + v8_cache_extra_paths: | + artifacts/native-engine-v8/linux-*/v8/buildtools/third_party/libc++ + artifacts/native-engine-v8/linux-*/v8/out/*/*/obj/buildtools/third_party/libc++abi/libc++abi.a + artifacts/native-engine-v8/linux-*/v8/third_party/libc++/src/include + artifacts/native-engine-v8/linux-*/v8/third_party/libc++abi/src/include + artifacts/native-engine-v8/linux-*/v8/third_party/llvm-build/Release+Asserts v8_cache_script: scripts/build-native-engine-runtime.sh v8_cache_patch: packaging/WebScene.NativeEngine.Runtime/patches/V8BuildNoCrelPatch.txt - os: windows-2022 rid: win-x64 + dotnet_architecture: x64 cpu: x64 monolith: v8_monolith.lib script: windows @@ -227,6 +346,7 @@ jobs: partition_alloc: true v8_configuration: ReleasePartitionAlloc v8_cache_generation: v9-v8-15.3.10-pa-windows-compat-inspector + v8_cache_extra_paths: '' v8_cache_script: scripts/build-native-engine-runtime.ps1 v8_cache_patch: packaging/WebScene.NativeEngine.Runtime/patches/V8WindowsCompatibilityPatch.txt runs-on: ${{ matrix.os }} @@ -246,25 +366,17 @@ jobs: with: dotnet-version: 8.0.x global-json-file: global.json + architecture: ${{ matrix.dotnet_architecture }} - name: Test V8 Windows child environment if: matrix.rid == 'win-x64' shell: pwsh run: ./scripts/tests/test_v8_windows_environment.ps1 - name: Expose the self-hosted Rust toolchain - if: matrix.rid == 'osx-arm64' + if: startsWith(matrix.rid, 'osx-') shell: bash run: | echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" "$HOME/.cargo/bin/cargo" --version - - name: Build Ubuntu 22.04 Linux runtime image - if: matrix.rid == 'linux-x64' - shell: bash - run: | - docker build \ - --platform linux/amd64 \ - --file packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-x64 \ - --tag webscene-native-linux-builder:ubuntu-22.04 \ - packaging/WebScene.NativeEngine.Runtime - id: v8-cache-key name: Resolve pinned V8 SDK cache identity shell: bash @@ -272,18 +384,21 @@ jobs: echo "image-version=${ImageVersion:-unknown}" >> "$GITHUB_OUTPUT" - id: v8-cache name: Restore pinned V8 SDK + if: github.ref_type != 'tag' uses: actions/cache/restore@v4 with: path: | artifacts/native-engine-v8/${{ matrix.rid }}/v8/include artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/args.gn artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/${{ matrix.monolith }} + artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++/libc++.a + ${{ matrix.v8_cache_extra_paths }} artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/icudtl.dat artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/gen/third_party/partition_alloc/src artifacts/native-engine-v8/${{ matrix.rid }}/v8/LICENSE artifacts/native-engine-v8/${{ matrix.rid }}/v8/third_party/icu/LICENSE artifacts/native-engine-v8/${{ matrix.rid }}/v8/third_party/partition_alloc/src - key: webscene-v8-sdk-${{ matrix.v8_cache_generation }}-${{ matrix.rid }}-${{ hashFiles('third-party/v8-patches/**') }}-${{ steps.v8-cache-key.outputs.image-version }}-${{ hashFiles(matrix.v8_cache_script, matrix.v8_cache_patch, 'scripts/V8WindowsEnvironment.psm1') }} + key: webscene-v8-sdk-${{ matrix.v8_cache_generation }}-${{ matrix.rid }}-${{ hashFiles('third-party/v8-patches/**') }}-${{ steps.v8-cache-key.outputs.image-version }}-${{ hashFiles(matrix.v8_cache_script, matrix.v8_cache_patch, 'scripts/V8WindowsEnvironment.psm1', 'packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocGlibc227Arm64Patch.txt', 'packaging/WebScene.NativeEngine.Runtime/patches/V8LibcxxMemoryResourcePatch.txt', 'packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc', 'packaging/WebScene.NativeEngine.Runtime/linux-build-lock.json', 'scripts/linux-glibc-toolchain.cmake') }} restore-keys: | ${{ matrix.rid != 'win-x64' && format('webscene-v8-sdk-{0}-{1}-{2}-', matrix.v8_cache_generation, matrix.rid, hashFiles('third-party/v8-patches/**')) || '' }} ${{ matrix.rid != 'win-x64' && format('webscene-v8-sdk-{0}-{1}-', matrix.v8_cache_generation, matrix.rid) || '' }} @@ -308,6 +423,25 @@ jobs: || { [[ -d "$root/third_party/partition_alloc/src" ]] \ && [[ -f "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/gen/third_party/partition_alloc/src/partition_alloc/buildflags.h" ]]; } } + libcxx_is_compatible() { + archive_is_regular() { + [[ -f "$1" ]] && [[ "$(head -c 7 "$1")" == '!' ]] + } + archive_has_memory_resource() { + "$root/third_party/llvm-build/Release+Asserts/bin/llvm-ar" t "$1" \ + | grep -Eq '(^|/)memory_resource\.o$' + } + [[ '${{ matrix.rid }}' != linux-* ]] \ + || { archive_is_regular "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++/libc++.a" \ + && archive_is_regular "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++abi/libc++abi.a" \ + && archive_has_memory_resource "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++/libc++.a" \ + && [[ -f "$root/buildtools/third_party/libc++/__config_site" ]] \ + && [[ -f "$root/buildtools/third_party/libc++/__assertion_handler" ]] \ + && [[ -f "$root/third_party/libc++/src/include/source_location" ]] \ + && [[ -f "$root/third_party/libc++abi/src/include/cxxabi.h" ]] \ + && [[ -x "$root/third_party/llvm-build/Release+Asserts/bin/clang++" ]] \ + && [[ -x "$root/third_party/llvm-build/Release+Asserts/bin/ld.lld" ]]; } + } args_are_compatible() { grep -Eq '^v8_enable_pointer_compression *= *true$' "$args" \ && grep -Eq '^v8_enable_pointer_compression_shared_cage *= *true$' "$args" \ @@ -316,10 +450,13 @@ jobs: || [[ '${{ matrix.rid }}' == 'win-x64' ]] \ || { grep -Eq '^use_allocator_shim *= *false$' "$args" \ && grep -Eq '^use_partition_alloc_as_malloc *= *false$' "$args"; }; } \ - && { [[ '${{ matrix.rid }}' != 'linux-x64' ]] \ + && { [[ '${{ matrix.rid }}' != linux-* ]] \ || { grep -Eq '^use_lld *= *true$' "$args" \ - && grep -Eq '^use_sysroot *= *false$' "$args" \ - && grep -Eq '^v8_monolithic_for_shared_library *= *true$' "$args"; }; } + && grep -Eq '^use_sysroot *= *true$' "$args" \ + && grep -Eq '^v8_monolithic_for_shared_library *= *true$' "$args" \ + && grep -Eq '^use_custom_libcxx *= *true$' "$args"; }; } \ + && { [[ '${{ matrix.rid }}' != osx-* ]] \ + || grep -Eq '^use_custom_libcxx *= *false$' "$args"; } } if [[ -f "$root/include/v8.h" \ && -f "$root/include/v8-inspector.h" \ @@ -330,7 +467,8 @@ jobs: && -f "$root/LICENSE" \ && -f "$root/third_party/icu/LICENSE" ]] \ && args_are_compatible \ - && partition_alloc_is_compatible; then + && partition_alloc_is_compatible \ + && libcxx_is_compatible; then echo "ready=true" >> "$GITHUB_OUTPUT" else echo "ready=false" >> "$GITHUB_OUTPUT" @@ -340,7 +478,7 @@ jobs: fi fi - name: Prepare pinned Dawn runtime dependency - if: matrix.rid != 'linux-x64' + if: matrix.rid == 'osx-arm64' || matrix.rid == 'win-x64' uses: ./.github/actions/graphics-sdk with: rid: ${{ matrix.rid }} @@ -354,7 +492,7 @@ jobs: - name: Build, pack, and test macOS runtime env: WEBSCENE_NATIVE_SKIP_HARDWARE_TESTS: '1' - if: matrix.rid == 'osx-arm64' + if: startsWith(matrix.rid, 'osx-') shell: bash run: | v8_root= @@ -362,9 +500,13 @@ jobs: if [[ '${{ steps.restored-v8-sdk.outputs.ready }}' == 'true' ]]; then v8_root="$restored_v8" fi + graphics_args=() + if [[ '${{ matrix.rid }}' == osx-arm64 ]]; then + graphics_args+=(--graphics-sdk "$GITHUB_WORKSPACE/artifacts/graphics-sdk/${{ matrix.rid }}") + fi scripts/build-native-engine-runtime.sh \ --rid '${{ matrix.rid }}' \ - --graphics-sdk "$GITHUB_WORKSPACE/artifacts/graphics-sdk/${{ matrix.rid }}" \ + "${graphics_args[@]}" \ --package-version '${{ needs.metadata.outputs.package-version }}' \ --v8-root "$v8_root" \ --v8-revision '${{ matrix.v8_revision }}' \ @@ -372,7 +514,7 @@ jobs: --partition-alloc \ --output "$GITHUB_WORKSPACE/artifacts/nuget-packages" - name: Upload exact macOS native symbols - if: matrix.rid == 'osx-arm64' + if: startsWith(matrix.rid, 'osx-') uses: actions/upload-artifact@v4 with: name: native-symbols-${{ matrix.rid }}-${{ needs.metadata.outputs.package-version }} @@ -380,36 +522,21 @@ jobs: if-no-files-found: error compression-level: 9 retention-days: 3 - - name: Build, pack, and test Linux runtime - if: matrix.rid == 'linux-x64' + - name: Build Linux runtime + if: startsWith(matrix.rid, 'linux-') shell: bash run: | - v8_root= - if [[ '${{ steps.restored-v8-sdk.outputs.ready }}' == 'true' ]]; then - v8_root="/workspace/artifacts/native-engine-v8/${{ matrix.rid }}/v8" + builder_args=() + if [[ -n '${{ needs.linux-builder.outputs.image }}' ]]; then + builder_args+=(--builder-image '${{ needs.linux-builder.outputs.image }}') + echo '${{ secrets.GITHUB_TOKEN }}' | docker login ghcr.io -u '${{ github.actor }}' --password-stdin fi - docker run --rm \ - --platform linux/amd64 \ - --user "$(id -u):$(id -g)" \ - --env HOME=/tmp/webscene-home \ - --env DOTNET_CLI_HOME=/tmp/webscene-home \ - --env CARGO_HOME=/tmp/webscene-home/.cargo \ - --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ - --volume "$GITHUB_WORKSPACE:/workspace" \ - --workdir /workspace \ - webscene-native-linux-builder:ubuntu-22.04 \ - bash -lc " - mkdir -p \"\$HOME\" - mkdir -p \"\$CARGO_HOME\" - scripts/build-native-engine-runtime-linux-container.sh \ - --rid '${{ matrix.rid }}' \ - --package-version '${{ needs.metadata.outputs.package-version }}' \ - --v8-root '$v8_root' \ - --v8-revision '${{ matrix.v8_revision }}' \ - --upstream-v8 \ - --partition-alloc \ - --output /workspace/artifacts/nuget-packages - " + scripts/build-linux-native-runtime.sh \ + --rid '${{ matrix.rid }}' \ + --package-version '${{ needs.metadata.outputs.package-version }}' \ + --output "$GITHUB_WORKSPACE/artifacts/nuget-packages" \ + --stage build \ + "${builder_args[@]}" - name: Build, pack, and test Windows runtime env: WEBSCENE_NATIVE_SKIP_HARDWARE_TESTS: '1' @@ -453,12 +580,13 @@ jobs: } Write-Host "Fresh Windows V8 checkout, GN generation, and Ninja output verified." - name: Run candidate compatibility discovery + if: matrix.rid != 'linux-arm64' continue-on-error: true shell: bash run: | case '${{ matrix.rid }}' in - osx-arm64) native_name=libwebscene_native_engine.dylib ;; - linux-x64) native_name=libwebscene_native_engine.so ;; + osx-arm64|osx-x64) native_name=libwebscene_native_engine.dylib ;; + linux-x64|linux-arm64) native_name=libwebscene_native_engine.so ;; win-x64) native_name=webscene_native_engine.dll ;; *) echo "Unsupported discovery RID '${{ matrix.rid }}'." >&2; exit 1 ;; esac @@ -472,13 +600,28 @@ jobs: exit 1 fi build_dir="${native_path%%/package-smoke/runtimes/*}" - dotnet run \ - --project tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj \ - -c Release --no-build -- \ - --selection candidate \ - --native-library "$native_path" \ - --native-cache-directory "$build_dir/code-cache" \ - --output "$build_dir/wpt-candidate-results" + runner_args=( + dotnet run + --project tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj + -c Release -f net10.0 --no-build --disable-build-servers -- + --selection candidate + --native-library "$native_path" + --native-cache-directory "$build_dir/code-cache" + --output "$build_dir/wpt-candidate-results") + if [[ '${{ matrix.rid }}' == 'linux-x64' ]]; then + docker run --rm \ + --platform linux/amd64 \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp/webscene-home \ + --env DOTNET_CLI_HOME=/tmp/webscene-home \ + --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ + --volume "$GITHUB_WORKSPACE:/workspace" \ + --workdir /workspace \ + '${{ needs.linux-builder.outputs.image }}' \ + "${runner_args[@]}" + else + "${runner_args[@]}" + fi - id: v8-sdk-ready name: Validate completed V8 SDK for caching if: always() && steps.v8-cache.outputs.cache-hit != 'true' @@ -492,16 +635,38 @@ jobs: || { [[ -d "$root/third_party/partition_alloc/src" ]] \ && [[ -f "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/gen/third_party/partition_alloc/src/partition_alloc/buildflags.h" ]]; } } + libcxx_is_compatible() { + archive_is_regular() { + [[ -f "$1" ]] && [[ "$(head -c 7 "$1")" == '!' ]] + } + archive_has_memory_resource() { + "$root/third_party/llvm-build/Release+Asserts/bin/llvm-ar" t "$1" \ + | grep -Eq '(^|/)memory_resource\.o$' + } + [[ '${{ matrix.rid }}' != linux-* ]] \ + || { archive_is_regular "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++/libc++.a" \ + && archive_is_regular "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++abi/libc++abi.a" \ + && archive_has_memory_resource "$root/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++/libc++.a" \ + && [[ -f "$root/buildtools/third_party/libc++/__config_site" ]] \ + && [[ -f "$root/buildtools/third_party/libc++/__assertion_handler" ]] \ + && [[ -f "$root/third_party/libc++/src/include/source_location" ]] \ + && [[ -f "$root/third_party/libc++abi/src/include/cxxabi.h" ]] \ + && [[ -x "$root/third_party/llvm-build/Release+Asserts/bin/clang++" ]] \ + && [[ -x "$root/third_party/llvm-build/Release+Asserts/bin/ld.lld" ]]; } + } args_are_compatible() { grep -Eq '^v8_enable_pointer_compression *= *true$' "$args" \ && grep -Eq '^v8_enable_pointer_compression_shared_cage *= *true$' "$args" \ && grep -Eq '^v8_enable_partition_alloc *= *${{ matrix.partition_alloc }}$' "$args" \ - && { [[ '${{ matrix.rid }}' != 'linux-x64' ]] \ + && { [[ '${{ matrix.rid }}' != linux-* ]] \ || { grep -Eq '^use_lld *= *true$' "$args" \ - && grep -Eq '^use_sysroot *= *false$' "$args" \ + && grep -Eq '^use_sysroot *= *true$' "$args" \ && grep -Eq '^v8_monolithic_for_shared_library *= *true$' "$args" \ + && grep -Eq '^use_custom_libcxx *= *true$' "$args" \ && grep -Eq '^use_allocator_shim *= *false$' "$args" \ - && grep -Eq '^use_partition_alloc_as_malloc *= *false$' "$args"; }; } + && grep -Eq '^use_partition_alloc_as_malloc *= *false$' "$args"; }; } \ + && { [[ '${{ matrix.rid }}' != osx-* ]] \ + || grep -Eq '^use_custom_libcxx *= *false$' "$args"; } } if [[ -f "$root/include/v8.h" \ && -f "$root/include/v8-inspector.h" \ @@ -512,56 +677,179 @@ jobs: && -f "$root/LICENSE" \ && -f "$root/third_party/icu/LICENSE" ]] \ && args_are_compatible \ - && partition_alloc_is_compatible; then + && partition_alloc_is_compatible \ + && libcxx_is_compatible; then echo "ready=true" >> "$GITHUB_OUTPUT" else echo "ready=false" >> "$GITHUB_OUTPUT" fi - name: Save completed V8 SDK - if: always() && steps.v8-sdk-ready.outputs.ready == 'true' + if: always() && github.ref_type != 'tag' && steps.v8-sdk-ready.outputs.ready == 'true' uses: actions/cache/save@v4 with: path: | artifacts/native-engine-v8/${{ matrix.rid }}/v8/include artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/args.gn artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/${{ matrix.monolith }} + artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/obj/buildtools/third_party/libc++/libc++.a + ${{ matrix.v8_cache_extra_paths }} artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/icudtl.dat artifacts/native-engine-v8/${{ matrix.rid }}/v8/out/${{ matrix.cpu }}/${{ matrix.v8_configuration }}/gen/third_party/partition_alloc/src artifacts/native-engine-v8/${{ matrix.rid }}/v8/LICENSE artifacts/native-engine-v8/${{ matrix.rid }}/v8/third_party/icu/LICENSE artifacts/native-engine-v8/${{ matrix.rid }}/v8/third_party/partition_alloc/src - key: webscene-v8-sdk-${{ matrix.v8_cache_generation }}-${{ matrix.rid }}-${{ hashFiles('third-party/v8-patches/**') }}-${{ steps.v8-cache-key.outputs.image-version }}-${{ hashFiles(matrix.v8_cache_script, matrix.v8_cache_patch, 'scripts/V8WindowsEnvironment.psm1') }} + key: webscene-v8-sdk-${{ matrix.v8_cache_generation }}-${{ matrix.rid }}-${{ hashFiles('third-party/v8-patches/**') }}-${{ steps.v8-cache-key.outputs.image-version }}-${{ hashFiles(matrix.v8_cache_script, matrix.v8_cache_patch, 'scripts/V8WindowsEnvironment.psm1', 'packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocGlibc227Arm64Patch.txt', 'packaging/WebScene.NativeEngine.Runtime/patches/V8LibcxxMemoryResourcePatch.txt', 'packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc', 'packaging/WebScene.NativeEngine.Runtime/linux-build-lock.json', 'scripts/linux-glibc-toolchain.cmake') }} - name: Upload required compatibility evidence - if: success() + if: success() && matrix.rid != 'linux-arm64' uses: actions/upload-artifact@v4 with: name: compatibility-required-${{ matrix.rid }}-${{ needs.metadata.outputs.package-version }} path: artifacts/native-engine-runtime-build/**/wpt-results/** if-no-files-found: error - name: Upload failed compatibility evidence - if: failure() + if: failure() && matrix.rid != 'linux-arm64' uses: actions/upload-artifact@v4 with: name: compatibility-required-${{ matrix.rid }}-${{ needs.metadata.outputs.package-version }} path: artifacts/native-engine-runtime-build/**/wpt-results/** if-no-files-found: warn - name: Upload candidate compatibility evidence - if: always() + if: always() && matrix.rid != 'linux-arm64' uses: actions/upload-artifact@v4 with: name: compatibility-candidate-${{ matrix.rid }}-${{ needs.metadata.outputs.package-version }} path: artifacts/native-engine-runtime-build/**/wpt-candidate-results/** if-no-files-found: warn - name: Upload verified RID package + if: matrix.rid != 'linux-arm64' uses: actions/upload-artifact@v4 with: name: native-${{ matrix.rid }}-${{ needs.metadata.outputs.package-version }} - path: artifacts/nuget-packages/*.nupkg + path: | + artifacts/nuget-packages/*.nupkg + artifacts/native-engine-runtime-build/**/*-abi.json + if-no-files-found: error + - name: Upload ARM64 cross-build stage + if: matrix.rid == 'linux-arm64' + uses: actions/upload-artifact@v4 + with: + name: cross-stage-linux-arm64-${{ needs.metadata.outputs.package-version }} + path: | + artifacts/native-engine-runtime-build/linux-arm64-*/** + artifacts/native-engine-v8/linux-arm64/v8/include/** + artifacts/native-engine-v8/linux-arm64/v8/out/arm64/ReleasePartitionAlloc/** + artifacts/native-engine-v8/linux-arm64/v8/LICENSE + artifacts/native-engine-v8/linux-arm64/v8/third_party/icu/LICENSE + artifacts/native-engine-v8/linux-arm64/v8/third_party/partition_alloc/src/** + if-no-files-found: error + retention-days: 3 + + linux-arm64-finalize: + name: Finalize and test linux-arm64 + needs: [metadata, linux-builder, native] + runs-on: [self-hosted, Linux, X64] + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Build ARM64 finalizer for PR validation + if: github.event_name == 'pull_request' + shell: bash + run: | + docker build \ + --platform linux/arm64 \ + --file packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-arm64-finalizer \ + --tag ghcr.io/scenetech/webscene-linux-arm64-finalizer:webscene-linux-glibc-v1 \ + packaging/WebScene.NativeEngine.Runtime + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Download ARM64 cross-build stage + uses: actions/download-artifact@v4 + with: + name: cross-stage-linux-arm64-${{ needs.metadata.outputs.package-version }} + path: artifacts + - name: Restore executable permissions + shell: bash + run: | + find artifacts/native-engine-runtime-build/linux-arm64-* -type f \ + \( -name 'webscene_*' -o -name '*_tests' \) -exec chmod +x {} + + - name: Generate snapshot, test, and package under ARM64 emulation + shell: bash + run: | + docker run --rm \ + --platform linux/arm64 \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp/webscene-home \ + --env DOTNET_CLI_HOME=/tmp/webscene-home \ + --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ + --volume "$GITHUB_WORKSPACE:/workspace" \ + --workdir /workspace \ + '${{ needs.linux-builder.outputs.arm64-finalizer-image }}' \ + scripts/build-linux-native-runtime.sh \ + --rid linux-arm64 \ + --package-version '${{ needs.metadata.outputs.package-version }}' \ + --output /workspace/artifacts/nuget-packages \ + --stage finalize + native_path="$(find artifacts/native-engine-runtime-build -path '*/linux-arm64-*/libwebscene_native_engine.so' -print -quit)" + python3 scripts/verify-linux-native-abi.py "$native_path" \ + --rid linux-arm64 \ + --output "${native_path%/*}/linux-arm64-abi.json" + - name: Run ARM64 candidate compatibility discovery + continue-on-error: true + shell: bash + run: | + native_path="$(find artifacts/native-engine-runtime-build -path '*/package-smoke/runtimes/linux-arm64/native/libwebscene_native_engine.so' -print -quit)" + build_dir="${native_path%%/package-smoke/runtimes/*}" + docker run --rm \ + --platform linux/arm64 \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp/webscene-home \ + --env DOTNET_CLI_HOME=/tmp/webscene-home \ + --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ + --volume "$GITHUB_WORKSPACE:/workspace" \ + --workdir /workspace \ + '${{ needs.linux-builder.outputs.arm64-finalizer-image }}' \ + dotnet run \ + --project tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj \ + -c Release -f net10.0 --no-build -- \ + --selection candidate \ + --native-library "$native_path" \ + --native-cache-directory "$build_dir/code-cache" \ + --output "$build_dir/wpt-candidate-results" + - name: Upload ARM64 required compatibility evidence + if: success() + uses: actions/upload-artifact@v4 + with: + name: compatibility-required-linux-arm64-${{ needs.metadata.outputs.package-version }} + path: artifacts/native-engine-runtime-build/**/wpt-results/** + if-no-files-found: error + - name: Upload ARM64 candidate compatibility evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: compatibility-candidate-linux-arm64-${{ needs.metadata.outputs.package-version }} + path: artifacts/native-engine-runtime-build/**/wpt-candidate-results/** + if-no-files-found: warn + - name: Upload verified ARM64 package and ABI evidence + uses: actions/upload-artifact@v4 + with: + name: native-linux-arm64-${{ needs.metadata.outputs.package-version }} + path: | + artifacts/nuget-packages/*.nupkg + artifacts/native-engine-runtime-build/**/*-abi.json if-no-files-found: error required-evidence: name: Verify cross-RID required evidence - needs: [metadata, native] + needs: [metadata, native, linux-arm64-finalize] runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 @@ -577,6 +865,8 @@ jobs: --profile tests/WebPlatformSubset/webscene-component-profile.json \ --selection required \ --expected-rid osx-arm64 \ + --expected-rid osx-x64 \ + --expected-rid linux-arm64 \ --expected-rid linux-x64 \ --expected-rid win-x64 \ --output artifacts/required-compatibility/cross-rid-summary.json @@ -589,7 +879,7 @@ jobs: package-set: name: Verify release package set - needs: [metadata, packages, native, required-evidence] + needs: [metadata, packages, native, linux-arm64-finalize, required-evidence] runs-on: [self-hosted, Linux, X64] steps: - uses: actions/checkout@v4 @@ -602,7 +892,7 @@ jobs: uses: actions/download-artifact@v4 with: pattern: native-*-${{ needs.metadata.outputs.package-version }} - path: artifacts/nuget-packages + path: artifacts merge-multiple: true - name: Verify versions, dependencies, symbols, and package inventory shell: bash @@ -611,6 +901,8 @@ jobs: artifacts/nuget-packages \ --version '${{ needs.metadata.outputs.package-version }}' \ --native-rid osx-arm64 \ + --native-rid osx-x64 \ + --native-rid linux-arm64 \ --native-rid linux-x64 \ --native-rid win-x64 \ --output artifacts/nuget-packages/release-packages.json @@ -631,7 +923,7 @@ jobs: candidate-evidence: name: Verify cross-RID candidate evidence - needs: [metadata, native] + needs: [metadata, native, linux-arm64-finalize] if: always() && needs.metadata.result == 'success' && needs.native.result != 'cancelled' runs-on: [self-hosted, Linux, X64] steps: @@ -651,6 +943,8 @@ jobs: --selection candidate \ --advisory-test-failures \ --expected-rid osx-arm64 \ + --expected-rid osx-x64 \ + --expected-rid linux-arm64 \ --expected-rid linux-x64 \ --expected-rid win-x64 \ --output artifacts/candidate-compatibility/cross-rid-summary.json @@ -664,17 +958,29 @@ jobs: consumer: name: Package consumer ${{ matrix.rid }} - needs: [metadata, package-set] + needs: [metadata, package-set, linux-builder] + permissions: + contents: read + packages: read strategy: fail-fast: false matrix: include: - os: [self-hosted, macOS, ARM64] rid: osx-arm64 + dotnet_architecture: arm64 + - os: [self-hosted, macOS, ARM64] + rid: osx-x64 + dotnet_architecture: x64 - os: [self-hosted, Linux, X64] rid: linux-x64 + dotnet_architecture: x64 + - os: [self-hosted, Linux, X64] + rid: linux-arm64 + dotnet_architecture: x64 - os: windows-2022 rid: win-x64 + dotnet_architecture: x64 runs-on: ${{ matrix.os }} env: # PR builds intentionally reuse package versions. Keep their extracted @@ -690,47 +996,206 @@ jobs: DOTNET_INSTALL_DIR: ${{ runner.temp }}/dotnet run: python3 scripts/prepare_dotnet_install.py "$DOTNET_INSTALL_DIR" - name: Setup .NET + if: matrix.rid != 'linux-arm64' uses: actions/setup-dotnet@v5 env: DOTNET_INSTALL_DIR: ${{ runner.temp }}/dotnet with: dotnet-version: 8.0.x global-json-file: global.json + architecture: ${{ matrix.dotnet_architecture }} + - name: Set up ARM64 emulation + if: matrix.rid == 'linux-arm64' + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Build ARM64 finalizer for PR consumer validation + if: github.event_name == 'pull_request' && matrix.rid == 'linux-arm64' + shell: bash + run: | + docker build \ + --platform linux/arm64 \ + --file packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-arm64-finalizer \ + --tag ghcr.io/scenetech/webscene-linux-arm64-finalizer:webscene-linux-glibc-v1 \ + packaging/WebScene.NativeEngine.Runtime + - name: Log in to GitHub Container Registry + if: matrix.rid == 'linux-arm64' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Download verified package set uses: actions/download-artifact@v4 with: name: release-${{ needs.metadata.outputs.package-version }} path: artifacts/nuget-packages - name: Restore clean package consumer + if: matrix.rid != 'linux-arm64' shell: bash run: | dotnet restore tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj \ + -p:TargetFramework=net10.0 \ -p:RuntimeIdentifier='${{ matrix.rid }}' \ -p:WebScenePackageVersion='${{ needs.metadata.outputs.package-version }}' \ - --configfile tests/WebScene.Runtime.PackageSmoke/NuGet.config + --configfile tests/WebScene.Runtime.PackageSmoke/NuGet.config \ + --disable-build-servers dotnet restore tests/WebScene.Uno.PackageSmoke/WebScene.Uno.PackageSmoke.csproj \ -p:RuntimeIdentifier='${{ matrix.rid }}' \ -p:WebScenePackageVersion='${{ needs.metadata.outputs.package-version }}' \ - --configfile tests/WebScene.Uno.PackageSmoke/NuGet.config + --configfile tests/WebScene.Uno.PackageSmoke/NuGet.config \ + --disable-build-servers - name: Build and run clean package consumer + if: matrix.rid != 'linux-arm64' shell: bash run: | dotnet run \ --project tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj \ - -c Release \ + -c Release -f net10.0 \ -r '${{ matrix.rid }}' \ --no-restore \ + --disable-build-servers \ -p:WebScenePackageVersion='${{ needs.metadata.outputs.package-version }}' dotnet run \ --project tests/WebScene.Uno.PackageSmoke/WebScene.Uno.PackageSmoke.csproj \ -c Release \ -r '${{ matrix.rid }}' \ --no-restore \ + --disable-build-servers \ -p:WebScenePackageVersion='${{ needs.metadata.outputs.package-version }}' + - name: Restore, build, and run ARM64 package consumer under emulation + if: matrix.rid == 'linux-arm64' + shell: bash + run: | + docker run --rm \ + --platform linux/arm64 \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp/webscene-home \ + --env DOTNET_CLI_HOME=/tmp/webscene-home \ + --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ + --env DOTNET_ROLL_FORWARD=Major \ + --volume "$GITHUB_WORKSPACE:/workspace" \ + --workdir /workspace \ + '${{ needs.linux-builder.outputs.arm64-finalizer-image }}' \ + bash -euo pipefail -c ' + dotnet restore tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj \ + -p:TargetFramework=net10.0 \ + -p:RuntimeIdentifier=linux-arm64 \ + -p:WebScenePackageVersion=${{ needs.metadata.outputs.package-version }} \ + --configfile tests/WebScene.Runtime.PackageSmoke/NuGet.config \ + --disable-build-servers + dotnet restore tests/WebScene.Uno.PackageSmoke/WebScene.Uno.PackageSmoke.csproj \ + -p:RuntimeIdentifier=linux-arm64 \ + -p:WebScenePackageVersion=${{ needs.metadata.outputs.package-version }} \ + --configfile tests/WebScene.Runtime.PackageSmoke/NuGet.config \ + --disable-build-servers + dotnet run --project tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj \ + -c Release -f net10.0 -r linux-arm64 --no-restore --disable-build-servers \ + -p:WebScenePackageVersion=${{ needs.metadata.outputs.package-version }} + dotnet run --project tests/WebScene.Uno.PackageSmoke/WebScene.Uno.PackageSmoke.csproj \ + -c Release -r linux-arm64 --no-restore --disable-build-servers \ + -p:WebScenePackageVersion=${{ needs.metadata.outputs.package-version }}' + + linux-floor-smoke: + name: Linux floor ${{ matrix.distribution }} ${{ matrix.rid }} + needs: [metadata, package-set] + strategy: + fail-fast: false + matrix: + include: + - os: [self-hosted, Linux, X64] + rid: linux-x64 + platform: linux/amd64 + distribution: ubuntu-18.04 + image: ubuntu:18.04 + - os: [self-hosted, Linux, X64] + rid: linux-arm64 + platform: linux/arm64 + distribution: ubuntu-18.04 + image: ubuntu:18.04 + - os: [self-hosted, Linux, X64] + rid: linux-x64 + platform: linux/amd64 + distribution: ubi-8.9 + image: registry.access.redhat.com/ubi8/ubi:8.9 + - os: [self-hosted, Linux, X64] + rid: linux-arm64 + platform: linux/arm64 + distribution: ubi-8.9 + image: registry.access.redhat.com/ubi8/ubi:8.9 + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up ARM64 emulation + if: matrix.platform == 'linux/arm64' + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - uses: actions/download-artifact@v4 + with: + name: release-${{ needs.metadata.outputs.package-version }} + path: artifacts/nuget-packages + - name: Extract native package + shell: bash + run: | + mkdir -p artifacts/linux-floor-smoke + unzip -q \ + "artifacts/nuget-packages/WebScene.NativeEngine.Runtime.${{ matrix.rid }}.${{ needs.metadata.outputs.package-version }}.nupkg" \ + -d artifacts/linux-floor-smoke + - name: Load native runtime on support floor + shell: bash + run: | + docker run --rm \ + --platform '${{ matrix.platform }}' \ + --volume "$GITHUB_WORKSPACE:/workspace:ro" \ + --workdir /workspace \ + '${{ matrix.image }}' \ + env LD_PRELOAD="/workspace/artifacts/linux-floor-smoke/runtimes/${{ matrix.rid }}/native/libwebscene_native_engine.so" \ + /bin/true + linux-compatibility-matrix: + name: Linux advisory ${{ matrix.distribution }} ${{ matrix.rid }} + needs: [metadata, package-set] + continue-on-error: true + strategy: + fail-fast: false + matrix: + include: + - { os: [self-hosted, Linux, X64], rid: linux-x64, platform: linux/amd64, distribution: debian-10, image: 'debian:10' } + - { os: [self-hosted, Linux, X64], rid: linux-arm64, platform: linux/arm64, distribution: debian-10, image: 'debian:10' } + - { os: [self-hosted, Linux, X64], rid: linux-x64, platform: linux/amd64, distribution: fedora-38, image: 'fedora:38' } + - { os: [self-hosted, Linux, X64], rid: linux-arm64, platform: linux/arm64, distribution: fedora-38, image: 'fedora:38' } + - { os: [self-hosted, Linux, X64], rid: linux-x64, platform: linux/amd64, distribution: opensuse-15.5, image: 'opensuse/leap:15.5' } + - { os: [self-hosted, Linux, X64], rid: linux-arm64, platform: linux/arm64, distribution: opensuse-15.5, image: 'opensuse/leap:15.5' } + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Set up ARM64 emulation + if: matrix.platform == 'linux/arm64' + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - uses: actions/download-artifact@v4 + with: + name: release-${{ needs.metadata.outputs.package-version }} + path: artifacts/nuget-packages + - name: Extract and load native runtime + shell: bash + run: | + mkdir -p artifacts/linux-advisory + unzip -q \ + "artifacts/nuget-packages/WebScene.NativeEngine.Runtime.${{ matrix.rid }}.${{ needs.metadata.outputs.package-version }}.nupkg" \ + -d artifacts/linux-advisory + docker run --rm \ + --platform '${{ matrix.platform }}' \ + --volume "$GITHUB_WORKSPACE:/workspace:ro" \ + --workdir /workspace \ + '${{ matrix.image }}' \ + env LD_PRELOAD="/workspace/artifacts/linux-advisory/runtimes/${{ matrix.rid }}/native/libwebscene_native_engine.so" \ + /bin/true publish: name: Publish to NuGet.org - needs: [metadata, consumer, release-ci-gate] + needs: [metadata, consumer, linux-floor-smoke, release-ci-gate] if: needs.metadata.outputs.publish == 'true' runs-on: [self-hosted, Linux, X64] environment: nuget.org @@ -781,3 +1246,67 @@ jobs: --source https://api.nuget.org/v3/index.json \ --skip-duplicate done + + linux-arm64-consumer-recheck: + name: Recheck package consumer linux-arm64 + if: inputs.consumer_recheck_run_id != '' + runs-on: [self-hosted, Linux, X64] + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v4 + - name: Set up ARM64 emulation + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + - name: Download verified package set + uses: actions/download-artifact@v4 + with: + name: release-${{ inputs.consumer_recheck_version }} + path: artifacts/nuget-packages + run-id: ${{ inputs.consumer_recheck_run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - id: image + name: Resolve pinned ARM64 .NET image + shell: bash + run: | + image="$(python3 - <<'PY' + import json + from pathlib import Path + lock = json.loads(Path("packaging/WebScene.NativeEngine.Runtime/linux-build-lock.json").read_text()) + finalizer = lock["arm64Finalizer"] + print(f'{finalizer["image"]}@{finalizer["digest"]}') + PY + )" + echo "reference=$image" >> "$GITHUB_OUTPUT" + - name: Restore, build, and run ARM64 package consumer + shell: bash + run: | + docker run --rm \ + --platform linux/arm64 \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp/webscene-home \ + --env DOTNET_CLI_HOME=/tmp/webscene-home \ + --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ + --volume "$GITHUB_WORKSPACE:/workspace" \ + --workdir /workspace \ + '${{ steps.image.outputs.reference }}' \ + bash -euo pipefail -c ' + dotnet restore tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj \ + -p:TargetFramework=net10.0 \ + -p:RuntimeIdentifier=linux-arm64 \ + -p:WebScenePackageVersion=${{ inputs.consumer_recheck_version }} \ + --configfile tests/WebScene.Runtime.PackageSmoke/NuGet.config \ + --disable-build-servers + dotnet restore tests/WebScene.Uno.PackageSmoke/WebScene.Uno.PackageSmoke.csproj \ + -p:RuntimeIdentifier=linux-arm64 \ + -p:WebScenePackageVersion=${{ inputs.consumer_recheck_version }} \ + --configfile tests/WebScene.Runtime.PackageSmoke/NuGet.config \ + --disable-build-servers + dotnet run --project tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj \ + -c Release -f net10.0 -r linux-arm64 --no-restore --disable-build-servers \ + -p:WebScenePackageVersion=${{ inputs.consumer_recheck_version }} + dotnet run --project tests/WebScene.Uno.PackageSmoke/WebScene.Uno.PackageSmoke.csproj \ + -c Release -r linux-arm64 --no-restore --disable-build-servers \ + -p:WebScenePackageVersion=${{ inputs.consumer_recheck_version }}' diff --git a/experiments/WebScene.NativeEngine.Probe/CMakeLists.txt b/experiments/WebScene.NativeEngine.Probe/CMakeLists.txt index ba9546b9c..3cef2689a 100644 --- a/experiments/WebScene.NativeEngine.Probe/CMakeLists.txt +++ b/experiments/WebScene.NativeEngine.Probe/CMakeLists.txt @@ -486,6 +486,8 @@ option(WEBSCENE_NATIVE_ENGINE_DENSE_LINK "Dead-strip unused native code and expose only the WebScene C ABI" OFF) option(WEBSCENE_NATIVE_ENGINE_THIN_LTO "Enable ThinLTO for the WebScene native engine and its V8 monolith link" OFF) +option(WEBSCENE_NATIVE_ENGINE_DEFER_TARGET_EXECUTION + "Build target executables without running snapshot generation or tests" OFF) option(WEBSCENE_NATIVE_ENGINE_CERTIFICATION "Include certification telemetry, diagnostic snapshots, and native profiling hooks" OFF) option(WEBSCENE_NATIVE_ENGINE_BUILD_HTML_PARSER_BENCHMARK @@ -619,12 +621,22 @@ if(WEBSCENE_NATIVE_ENGINE_HTML_PARSER STREQUAL "html5ever") "${CMAKE_CURRENT_SOURCE_DIR}/native/html_parser/Cargo.toml") set(WEBSCENE_HTML_PARSER_TARGET_DIR "${CMAKE_CURRENT_BINARY_DIR}/html-parser-target") + set(WEBSCENE_HTML_PARSER_LIBRARY_DIR + "${WEBSCENE_HTML_PARSER_TARGET_DIR}/release") + set(WEBSCENE_HTML_PARSER_CARGO_TARGET_ARGS) + if(DEFINED WEBSCENE_RUST_TARGET_TRIPLE + AND NOT WEBSCENE_RUST_TARGET_TRIPLE STREQUAL "") + set(WEBSCENE_HTML_PARSER_LIBRARY_DIR + "${WEBSCENE_HTML_PARSER_TARGET_DIR}/${WEBSCENE_RUST_TARGET_TRIPLE}/release") + list(APPEND WEBSCENE_HTML_PARSER_CARGO_TARGET_ARGS + --target "${WEBSCENE_RUST_TARGET_TRIPLE}") + endif() if(MSVC) set(WEBSCENE_HTML_PARSER_LIBRARY - "${WEBSCENE_HTML_PARSER_TARGET_DIR}/release/webscene_html_parser.lib") + "${WEBSCENE_HTML_PARSER_LIBRARY_DIR}/webscene_html_parser.lib") else() set(WEBSCENE_HTML_PARSER_LIBRARY - "${WEBSCENE_HTML_PARSER_TARGET_DIR}/release/libwebscene_html_parser.a") + "${WEBSCENE_HTML_PARSER_LIBRARY_DIR}/libwebscene_html_parser.a") endif() add_custom_command( OUTPUT "${WEBSCENE_HTML_PARSER_LIBRARY}" @@ -632,7 +644,9 @@ if(WEBSCENE_NATIVE_ENGINE_HTML_PARSER STREQUAL "html5ever") "CARGO_TARGET_DIR=${WEBSCENE_HTML_PARSER_TARGET_DIR}" "${WEBSCENE_CARGO_EXECUTABLE}" build --manifest-path "${WEBSCENE_HTML_PARSER_MANIFEST}" - --release --locked ${WEBSCENE_HTML_PARSER_CARGO_FEATURES} + --release --locked + ${WEBSCENE_HTML_PARSER_CARGO_TARGET_ARGS} + ${WEBSCENE_HTML_PARSER_CARGO_FEATURES} DEPENDS "${WEBSCENE_HTML_PARSER_MANIFEST}" "${CMAKE_CURRENT_SOURCE_DIR}/native/html_parser/Cargo.lock" @@ -1030,19 +1044,24 @@ if(WEBSCENE_NATIVE_ENGINE_ENABLE_V8) "${CMAKE_CURRENT_BINARY_DIR}/webscene_bootstrap_snapshot.bin") set(WEBSCENE_V8_SNAPSHOT_METADATA "${CMAKE_CURRENT_BINARY_DIR}/webscene_bootstrap_snapshot.meta") - add_custom_command( - OUTPUT "${WEBSCENE_V8_SNAPSHOT_BLOB}" "${WEBSCENE_V8_SNAPSHOT_METADATA}" - COMMAND "$" - "${WEBSCENE_V8_ICU_DATA}" - "${WEBSCENE_V8_BOOTSTRAP_SOURCE}" - "${WEBSCENE_V8_SNAPSHOT_BLOB}" - "${WEBSCENE_V8_SNAPSHOT_METADATA}" - DEPENDS webscene_v8_snapshot_builder "${WEBSCENE_V8_BOOTSTRAP_SOURCE}" - COMMENT "Creating the WebScene V8 bootstrap snapshot" - VERBATIM) - add_custom_target(webscene_v8_bootstrap_snapshot ALL - DEPENDS "${WEBSCENE_V8_SNAPSHOT_BLOB}" "${WEBSCENE_V8_SNAPSHOT_METADATA}") - add_dependencies(webscene_native_engine webscene_v8_bootstrap_snapshot) + if(NOT WEBSCENE_NATIVE_ENGINE_DEFER_TARGET_EXECUTION) + add_custom_command( + OUTPUT "${WEBSCENE_V8_SNAPSHOT_BLOB}" "${WEBSCENE_V8_SNAPSHOT_METADATA}" + COMMAND "$" + "${WEBSCENE_V8_ICU_DATA}" + "${WEBSCENE_V8_BOOTSTRAP_SOURCE}" + "${WEBSCENE_V8_SNAPSHOT_BLOB}" + "${WEBSCENE_V8_SNAPSHOT_METADATA}" + DEPENDS webscene_v8_snapshot_builder "${WEBSCENE_V8_BOOTSTRAP_SOURCE}" + COMMENT "Creating the WebScene V8 bootstrap snapshot" + VERBATIM) + add_custom_target(webscene_v8_bootstrap_snapshot ALL + DEPENDS "${WEBSCENE_V8_SNAPSHOT_BLOB}" "${WEBSCENE_V8_SNAPSHOT_METADATA}") + add_dependencies(webscene_native_engine webscene_v8_bootstrap_snapshot) + else() + message(STATUS + "WebScene native engine: target execution deferred for cross-build finalization") + endif() target_compile_definitions(webscene_native_engine PRIVATE WEBSCENE_V8_BOOTSTRAP_SNAPSHOT=1 WEBSCENE_V8_SNAPSHOT_FILENAME="webscene_bootstrap_snapshot.bin" @@ -1267,6 +1286,7 @@ if(WEBSCENE_NATIVE_ENGINE_ENABLE_V8) ENVIRONMENT "WEBSCENE_NATIVE_ENGINE_TEST_FILTER=host-driven-close-veto" LABELS "native;runtime;lifecycle;performance" TIMEOUT 30) set_tests_properties(webscene_native_engine_tests PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../.." ENVIRONMENT "WEBSCENE_V8_DETAILED_MEMORY_METRICS=1;WEBSCENE_INTEROP_STRESS=1") endif() diff --git a/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime.cpp b/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime.cpp index 835b5a5e0..267199149 100644 --- a/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime.cpp +++ b/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime.cpp @@ -96,6 +96,7 @@ #include #include #include +#include #include #include #include diff --git a/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_canvas.inc b/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_canvas.inc index 788bb26ce..5008eb6ae 100644 --- a/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_canvas.inc +++ b/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_canvas.inc @@ -2390,10 +2390,17 @@ break; } const auto number = shorthand.substr(number_start, index - number_start); - const std::string number_text{number}; - char* parsed_end = nullptr; - const auto parsed_size = std::strtof(number_text.c_str(), &parsed_end); - if (parsed_end == number_text.c_str() + number_text.size() + // Floating-point std::from_chars is only available from macOS + // 26 in the current Apple SDK. strtof is available on the + // macOS 14 deployment baseline; copy the bounded view so its + // end can still be validated exactly. + const std::string number_text(number); + char* parse_end = nullptr; + errno = 0; + const auto parsed_size = std::strtof(number_text.c_str(), &parse_end); + if (parse_end == number_text.c_str() + number_text.size() + && parse_end != number_text.c_str() + && errno != ERANGE && std::isfinite(parsed_size) && parsed_size > 0.0F) { result.size = parsed_size; diff --git a/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_support.inc b/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_support.inc index 91267be03..c77b2db5c 100644 --- a/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_support.inc +++ b/experiments/WebScene.NativeEngine.Probe/native/webscene_v8_runtime_support.inc @@ -954,6 +954,10 @@ void install_navigator( constexpr auto platform = "Win32"; constexpr auto user_agent_platform = "Windows NT 10.0; Win64; x64"; constexpr auto client_platform = "Windows"; +#elif defined(__aarch64__) + constexpr auto platform = "Linux aarch64"; + constexpr auto user_agent_platform = "X11; Linux aarch64"; + constexpr auto client_platform = "Linux"; #else constexpr auto platform = "Linux x86_64"; constexpr auto user_agent_platform = "X11; Linux x86_64"; diff --git a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_browser_dom_tests.inc b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_browser_dom_tests.inc index e9daa7a3a..dbc25a2e9 100644 --- a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_browser_dom_tests.inc +++ b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_browser_dom_tests.inc @@ -4638,7 +4638,7 @@ void test_async_save_acknowledgement_publishes_without_pointer_input() state.pending = false; state.acknowledgements++; if (!shouldFail) label.textContent = ''; - }, 250); + }, 1000); }); })() )JS", "async-save-publication-setup.js"); @@ -4653,6 +4653,21 @@ void test_async_save_acknowledgement_publishes_without_pointer_input() const auto consumed_before = consumed_input_count(engine); pointer_button(engine, WEBSCENE_INPUT_POINTER_DOWN, 30, 20, 9102U, true); pointer_button(engine, WEBSCENE_INPUT_POINTER_UP, 30, 20, 9103U, false); + wait_for_consumed_inputs( + engine, + consumed_before + 2U, + "initial async-save activation was not consumed"); + const auto initial_state = evaluate_until_equals(engine, R"JS(({ activations: __saveState.activations, + requests: __saveState.requests, + pending: __saveState.pending + }))JS", "async-save-initial-activation.js", + R"JSON({"activations":1,"requests":1,"pending":true})JSON", + 1000); + require( + initial_state + == R"JSON({"activations":1,"requests":1,"pending":true})JSON", + "initial activation did not start an asynchronous save: " + + initial_state); pointer_button(engine, WEBSCENE_INPUT_POINTER_DOWN, 30, 20, 9104U, true); pointer_button(engine, WEBSCENE_INPUT_POINTER_UP, 30, 20, 9105U, false); wait_for_consumed_inputs( @@ -4665,7 +4680,8 @@ void test_async_save_acknowledgement_publishes_without_pointer_input() pending: __saveState.pending, label: document.getElementById('label').textContent }))JS", "async-save-coalescing.js", - R"JSON({"activations":2,"requests":1,"pending":true,"label":"Save"})JSON"); + R"JSON({"activations":2,"requests":1,"pending":true,"label":"Save"})JSON", + 1000); require( coalesced_state == R"JSON({"activations":2,"requests":1,"pending":true,"label":"Save"})JSON", @@ -4675,7 +4691,7 @@ void test_async_save_acknowledgement_publishes_without_pointer_input() webscene_engine_metrics after_activation{}; webscene_engine_get_metrics(engine, &after_activation); webscene_engine_metrics after_ack{}; - for (auto attempt = 0; attempt < 200; ++attempt) { + for (auto attempt = 0; attempt < 1000; ++attempt) { webscene_engine_get_metrics(engine, &after_ack); if (after_ack.published_scenes > after_activation.published_scenes && evaluate(engine, "__saveState.acknowledgements", @@ -4712,7 +4728,7 @@ void test_async_save_acknowledgement_publishes_without_pointer_input() engine, failure_consumed_before + 2U, "failing save activation was not consumed"); - for (auto attempt = 0; attempt < 200; ++attempt) { + for (auto attempt = 0; attempt < 1000; ++attempt) { if (evaluate(engine, "__saveState.acknowledgements", "async-save-failure-wait.js") == "2") { break; diff --git a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_canvas_tests.inc b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_canvas_tests.inc index b0dbcdf40..37c163a2d 100644 --- a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_canvas_tests.inc +++ b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_canvas_tests.inc @@ -167,6 +167,8 @@ void test_canvas_text_metrics_use_host_font_axes() const small = context.measureText('MMMM'); context.font = '20px sans-serif'; const large = context.measureText('MMMM'); + context.font = '12.5px sans-serif'; + const fractional = context.measureText('MMMM'); context.font = "bold 12px -apple-system, BlinkMacSystemFont, 'Trebuchet MS', sans-serif"; const bold = context.measureText('Label'); context.textBaseline = 'middle'; @@ -180,6 +182,7 @@ void test_canvas_text_metrics_use_host_font_axes() bold.fontBoundingBoxDescent ].every(Number.isFinite), sizeChangesAdvance: large.width > small.width * 1.9, + fractionalSizeParsed: fractional.fontBoundingBoxAscent === 9.375, ascent: bold.actualBoundingBoxAscent, descent: bold.actualBoundingBoxDescent, fontAscent: bold.fontBoundingBoxAscent, @@ -195,7 +198,8 @@ void test_canvas_text_metrics_use_host_font_axes() require( result.find("\"defaultFieldsAreFinite\":true") != std::string::npos - && result.find("\"sizeChangesAdvance\":true") != std::string::npos, + && result.find("\"sizeChangesAdvance\":true") != std::string::npos + && result.find("\"fractionalSizeParsed\":true") != std::string::npos, "Canvas TextMetrics did not use finite host font metrics: " + result); require( result.find("\"ascent\":7.2") != std::string::npos @@ -212,7 +216,7 @@ void test_canvas_text_metrics_use_host_font_axes() "Canvas TextMetrics did not resolve distances from the active middle baseline: " + result); require( - probe.calls == 3U, + probe.calls == 4U, "Canvas measureText did not reuse host metrics across baseline-only changes"); require(probe.text == "Label", "Canvas measureText changed the measured text"); require( diff --git a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_input_tests.inc b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_input_tests.inc index 28ada9ae0..4fc4e4454 100644 --- a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_input_tests.inc +++ b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_input_tests.inc @@ -2378,6 +2378,10 @@ void test_navigator_platform_and_wheel_modifiers(webscene_engine* engine) require( navigator_result.find(R"("platform":"Win32")") != std::string::npos, "native navigator did not expose Windows platform identity: " + navigator_result); +#elif defined(__aarch64__) + require( + navigator_result.find(R"("platform":"Linux aarch64")") != std::string::npos, + "native navigator did not expose Linux ARM64 platform identity: " + navigator_result); #else require( navigator_result.find(R"("platform":"Linux x86_64")") != std::string::npos, diff --git a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_inspector_tests.inc b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_inspector_tests.inc index 7fdf0097e..3efa78167 100644 --- a/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_inspector_tests.inc +++ b/experiments/WebScene.NativeEngine.Probe/tests/native_v8_runtime_inspector_tests.inc @@ -61,12 +61,13 @@ void receive_inspector_test_message( bool wait_for_inspector_message( inspector_test_messages& messages, std::string_view first, - std::string_view second = {}) + std::string_view second = {}, + std::chrono::steady_clock::duration timeout = std::chrono::seconds(5)) { std::unique_lock lock(messages.mutex); return messages.available.wait_for( lock, - std::chrono::seconds(5), + timeout, [&] { return std::any_of( messages.values.begin(), @@ -1391,7 +1392,11 @@ void test_v8_inspector_raw_cdp_session(webscene_engine* engine) session_id, R"({"id":68,"method":"Runtime.evaluate","params":{"expression":"globalThis.__websceneCappedRejections = Array.from({ length: 1025 }, (_, index) => Promise.reject(new Error('webscene-inspector-capped-rejection-' + index)));","returnByValue":true}})"); require( - wait_for_inspector_message(messages, R"("id":68)"), + wait_for_inspector_message( + messages, + R"("id":68)", + {}, + std::chrono::seconds(30)), "Inspector rejection-cap fixture did not evaluate"); require( wait_for_inspector_message_count( diff --git a/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-arm64-finalizer b/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-arm64-finalizer new file mode 100644 index 000000000..725fe31a4 --- /dev/null +++ b/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-arm64-finalizer @@ -0,0 +1,17 @@ +FROM mcr.microsoft.com/dotnet/sdk:10.0.302-noble-arm64v8@sha256:683d16913974bf1311381ccd6d6aba55213f313501c39ef964b0458f44c0c4bc + +ENV DEBIAN_FRONTEND=noninteractive \ + DOTNET_CLI_TELEMETRY_OPTOUT=1 \ + DOTNET_NOLOGO=1 \ + NUGET_XMLDOC_MODE=skip + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + binutils=2.42-4ubuntu2.10 \ + cmake=3.28.3-1build7 \ + fontconfig=2.15.0-1.1ubuntu2 \ + fonts-dejavu-core=2.37-8 \ + python3=3.12.3-0ubuntu2.1 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /workspace diff --git a/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc b/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc new file mode 100644 index 000000000..46b9d30cc --- /dev/null +++ b/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc @@ -0,0 +1,48 @@ +FROM mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net10.0-cross-amd64@sha256:2962cae8ca49b18fb533504513c89308927ed3721372a0cc58630c350a2936b8 AS x64-sysroot +FROM mcr.microsoft.com/dotnet-buildtools/prereqs:azurelinux-3.0-net10.0-cross-arm64@sha256:619e1c013b88c504d34c8e064e0860313cebeb3ee1fc6e2e838406744c9857a8 AS arm64-sysroot +FROM mcr.microsoft.com/dotnet/sdk:10.0.302-noble-amd64@sha256:7a91ccecc26d71bf7688c627a6b5eae2e27bb2cd1e37e8abe738348904245692 AS dotnet-sdk +FROM x64-sysroot + +ARG RUST_VERSION=1.90.0 +ARG RUST_ARCHIVE_SHA256=bff8974f2d3ee6c0e6ac926b533f65bbdd3697d2c2b925bdae5f45b9eed10a67 +ARG RUST_ARM64_STD_SHA256=4952abb7d9d3ed7cea4f7ea44dcb23dc67631fae4ac44a5f059b90a4b5e9223f +ARG DEPOT_TOOLS_COMMIT=ca054941f756b50e1a3d83727270d879bec1f331 + +ENV DEBIAN_FRONTEND=noninteractive \ + DOTNET_ROOT=/usr/share/dotnet \ + DOTNET_CLI_TELEMETRY_OPTOUT=1 \ + DOTNET_NOLOGO=1 \ + NUGET_XMLDOC_MODE=skip \ + DEPOT_TOOLS_UPDATE=0 \ + PATH=/opt/depot_tools:/opt/rust/bin:/usr/share/dotnet:${PATH} + +COPY --from=arm64-sysroot /crossrootfs/arm64 /crossrootfs/arm64 +COPY --from=dotnet-sdk /usr/share/dotnet /usr/share/dotnet + +RUN tdnf install -y \ + dejavu-sans-fonts-2.37-3.azl3 \ + dejavu-sans-mono-fonts-2.37-3.azl3 \ + dejavu-serif-fonts-2.37-3.azl3 \ + fontconfig-2.14.2-2.azl3 \ + && tdnf clean all + +RUN set -eux; \ + curl -fsSLO "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-x86_64-unknown-linux-gnu.tar.xz"; \ + echo "${RUST_ARCHIVE_SHA256} rust-${RUST_VERSION}-x86_64-unknown-linux-gnu.tar.xz" | sha256sum -c -; \ + tar -xf "rust-${RUST_VERSION}-x86_64-unknown-linux-gnu.tar.xz"; \ + "rust-${RUST_VERSION}-x86_64-unknown-linux-gnu/install.sh" --prefix=/opt/rust --without=rust-docs; \ + curl -fsSLO "https://static.rust-lang.org/dist/rust-std-${RUST_VERSION}-aarch64-unknown-linux-gnu.tar.xz"; \ + echo "${RUST_ARM64_STD_SHA256} rust-std-${RUST_VERSION}-aarch64-unknown-linux-gnu.tar.xz" | sha256sum -c -; \ + tar -xf "rust-std-${RUST_VERSION}-aarch64-unknown-linux-gnu.tar.xz"; \ + "rust-std-${RUST_VERSION}-aarch64-unknown-linux-gnu/install.sh" --prefix=/opt/rust; \ + rm -rf rust-*.tar.xz rust-*unknown-linux-gnu + +RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git /opt/depot_tools \ + && git -C /opt/depot_tools checkout --detach "$DEPOT_TOOLS_COMMIT" \ + && test "$(git -C /opt/depot_tools rev-parse HEAD)" = "$DEPOT_TOOLS_COMMIT" \ + && /opt/depot_tools/ensure_bootstrap \ + && git config --system --add safe.directory /opt/depot_tools \ + && git config --system --add safe.directory /opt/depot_tools/.git + +COPY linux-build-lock.json /opt/webscene/linux-build-lock.json +WORKDIR /workspace diff --git a/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-x64 b/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-x64 deleted file mode 100644 index 7ef866999..000000000 --- a/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-x64 +++ /dev/null @@ -1,58 +0,0 @@ -# syntax=docker/dockerfile:1 - -FROM mcr.microsoft.com/dotnet/sdk:10.0.302-noble-amd64 AS dotnet-sdk -FROM mcr.microsoft.com/dotnet/runtime:8.0-jammy-amd64 AS dotnet8-runtime - -# Build the distributable runtime against Ubuntu 22.04's glibc 2.35 rather -# than the newer libc provided by the current GitHub-hosted runner image. -FROM ubuntu:22.04 - -ENV DEBIAN_FRONTEND=noninteractive \ - DOTNET_ROOT=/usr/share/dotnet \ - PATH=/usr/share/dotnet:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - CC=gcc-12 \ - CXX=g++-12 \ - DOTNET_CLI_TELEMETRY_OPTOUT=1 \ - DOTNET_NOLOGO=1 \ - NUGET_XMLDOC_MODE=skip - -RUN apt-get update \ - && apt-get install --yes --no-install-recommends \ - build-essential \ - ca-certificates \ - clang \ - cmake \ - curl \ - fonts-dejavu-core \ - gdb \ - gcc-12 \ - g++-12 \ - git \ - libfontconfig1 \ - libglib2.0-dev \ - libssl-dev \ - lld \ - ninja-build \ - pkg-config \ - python3 \ - unzip \ - xz-utils \ - zlib1g-dev \ - && rm -rf /var/lib/apt/lists/* - -# html5ever is compiled into the existing WebScene DSO. Keep the Rust compiler -# pinned independently from Ubuntu's older distro package so native release -# builds resolve the same Cargo.lock on every host. Install it in /opt because -# CI deliberately runs this image as the host's non-root UID. -ENV RUSTUP_HOME=/opt/rustup \ - CARGO_HOME=/opt/cargo \ - PATH=/opt/cargo/bin:${PATH} -RUN mkdir -p "$RUSTUP_HOME" "$CARGO_HOME" \ - && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --profile minimal --default-toolchain 1.90.0 \ - && chmod -R a+rX "$RUSTUP_HOME" "$CARGO_HOME" - -COPY --from=dotnet-sdk /usr/share/dotnet /usr/share/dotnet -COPY --from=dotnet8-runtime /usr/share/dotnet/shared/Microsoft.NETCore.App /usr/share/dotnet/shared/Microsoft.NETCore.App - -WORKDIR /workspace diff --git a/packaging/WebScene.NativeEngine.Runtime/README.md b/packaging/WebScene.NativeEngine.Runtime/README.md index 4c352d2b4..9f5b69f3c 100644 --- a/packaging/WebScene.NativeEngine.Runtime/README.md +++ b/packaging/WebScene.NativeEngine.Runtime/README.md @@ -63,15 +63,36 @@ Install the package matching the application's deployment RID: ```xml + + ``` | Target platform | Runtime identifier | Package | | --- | --- | --- | | macOS on Apple silicon | `osx-arm64` | [`WebScene.NativeEngine.Runtime.osx-arm64`](https://www.nuget.org/packages/WebScene.NativeEngine.Runtime.osx-arm64/) | +| macOS on Intel | `osx-x64` | [`WebScene.NativeEngine.Runtime.osx-x64`](https://www.nuget.org/packages/WebScene.NativeEngine.Runtime.osx-x64/) | | Linux x64 | `linux-x64` | [`WebScene.NativeEngine.Runtime.linux-x64`](https://www.nuget.org/packages/WebScene.NativeEngine.Runtime.linux-x64/) | +| Linux ARM64 | `linux-arm64` | [`WebScene.NativeEngine.Runtime.linux-arm64`](https://www.nuget.org/packages/WebScene.NativeEngine.Runtime.linux-arm64/) | | Windows x64 | `win-x64` | [`WebScene.NativeEngine.Runtime.win-x64`](https://www.nuget.org/packages/WebScene.NativeEngine.Runtime.win-x64/) | Additional RIDs listed by the package definition are reserved until their release lanes are enabled. + +## Reproducible Linux builds + +Linux packages use the immutable inputs recorded in `linux-build-lock.json` and +the .NET-style x64 cross-builder in `Dockerfile.linux-glibc`. Build either glibc +RID locally with the same entry point used by CI: + +```bash +scripts/build-linux-native-runtime.sh --rid linux-x64 --package-version VERSION +scripts/build-linux-native-runtime.sh --rid linux-arm64 --package-version VERSION +``` + +The ARM64 command creates a cross-build stage. CI transfers that stage to an +x64 runner with ARM64 emulation and calls the same command with `--stage finalize` +to create the V8 bootstrap snapshot, execute tests, and pack the NuGet package. Published +Linux binaries must pass `verify-linux-native-abi.py`, including the glibc 2.27, +GLIBCXX, CXXABI, dependency, architecture, RPATH, and exported-ABI gates. diff --git a/packaging/WebScene.NativeEngine.Runtime/WebScene.NativeEngine.Runtime.csproj b/packaging/WebScene.NativeEngine.Runtime/WebScene.NativeEngine.Runtime.csproj index 565f598c8..6f4464e52 100644 --- a/packaging/WebScene.NativeEngine.Runtime/WebScene.NativeEngine.Runtime.csproj +++ b/packaging/WebScene.NativeEngine.Runtime/WebScene.NativeEngine.Runtime.csproj @@ -11,7 +11,9 @@ WebScene.NativeEngine.Runtime.Template $(WebSceneNativeEngineRid) macOS on Apple silicon + macOS on Intel Linux x64 + Linux ARM64 Windows x64 WebScene Native Engine Runtime for $(WebSceneNativeEnginePlatformName) Native WebScene V8, DOM, CSS, layout, Canvas, SVG, and immutable-scene runtime for $(WebSceneNativeEnginePlatformName) ($(WebSceneNativeEngineRid)), built to run trusted web-authored UI without a WebView or embedded browser. @@ -36,6 +38,9 @@ generated bootstrap Release + + + osx-arm64;osx-x64;linux-arm64;linux-x64;win-arm64;win-x64 libwebscene_native_engine.dylib libwebscene_native_engine.so @@ -272,7 +277,7 @@ ,"architecture":"$(WebSceneNativeEngineArchitecture)","importLibraryFileName":"$(WebSceneNativeEngineExpectedImportLibraryFileName)","importLibrarySha256":"$(WebSceneNativeEngineImportLibraryHash)" ) + #define HAS_HW_CAPS + #endif + diff --git a/scripts/build-linux-native-runtime.sh b/scripts/build-linux-native-runtime.sh new file mode 100755 index 000000000..c8ae9d0a8 --- /dev/null +++ b/scripts/build-linux-native-runtime.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +lock_file="$repo_root/packaging/WebScene.NativeEngine.Runtime/linux-build-lock.json" +dockerfile="$repo_root/packaging/WebScene.NativeEngine.Runtime/Dockerfile.linux-glibc" +rid= +package_version= +output_dir="$repo_root/artifacts/nuget-packages" +stage=build +builder_image= + +usage() { + echo "Usage: $0 --rid linux-x64|linux-arm64 [--package-version VERSION] [--output DIR] [--stage build|finalize] [--builder-image IMAGE@sha256:DIGEST]" >&2 +} + +while (($# > 0)); do + case "$1" in + --rid) rid="${2:-}"; shift 2 ;; + --package-version) package_version="${2:-}"; shift 2 ;; + --output) output_dir="${2:-}"; shift 2 ;; + --stage) stage="${2:-}"; shift 2 ;; + --builder-image) builder_image="${2:-}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "Unknown option: $1" >&2; usage; exit 1 ;; + esac +done + +case "$rid" in + linux-x64|linux-arm64) ;; + *) usage; exit 1 ;; +esac +case "$stage" in + build|finalize) ;; + *) usage; exit 1 ;; +esac + +IFS='|' read -r builder_identity target_triple rust_target_triple sysroot max_glibc max_glibcxx max_cxxabi <<< "$(python3 - "$lock_file" "$rid" <<'PY' +import json, pathlib, sys +lock = json.loads(pathlib.Path(sys.argv[1]).read_text()) +target = lock["sysroots"][sys.argv[2]] +print("|".join(( + lock["builderIdentity"], target["targetTriple"], target["rustTargetTriple"], target["path"], + lock["compatibility"]["maximumGlibc"], + lock["compatibility"]["maximumGlibcxx"], + lock["compatibility"]["maximumCxxabi"], +))) +PY +)" +cargo_target_key="$(printf '%s' "$rust_target_triple" | tr '[:lower:]-' '[:upper:]_')" +cargo_linker_name="CARGO_TARGET_${cargo_target_key}_LINKER" +cargo_rustflags_name="CARGO_TARGET_${cargo_target_key}_RUSTFLAGS" +source_date_epoch="$(git -C "$repo_root" show -s --format=%ct HEAD)" +git_common_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir)" +docker_mount_args=(--volume "$repo_root:/workspace") +if [[ "$git_common_dir" != "$repo_root/.git" ]]; then + docker_mount_args+=(--volume "$git_common_dir:$git_common_dir:ro") +fi + +if [[ "$stage" == finalize ]]; then + "$repo_root/scripts/build-native-engine-runtime.sh" \ + --rid "$rid" \ + --target-triple "$target_triple" \ + --rust-target-triple "$rust_target_triple" \ + --sysroot "$sysroot" \ + --builder-identity "$builder_identity" \ + --glibc-baseline "$max_glibc" \ + --package-version "$package_version" \ + --partition-alloc \ + --upstream-v8 \ + --output "$output_dir" \ + --finalize-only + exit 0 +fi + +if [[ -z "$builder_image" ]]; then + builder_image="webscene-linux-builder:$builder_identity" + docker build \ + --platform linux/amd64 \ + --file "$dockerfile" \ + --tag "$builder_image" \ + "$repo_root/packaging/WebScene.NativeEngine.Runtime" +elif [[ "$builder_image" != *@sha256:* ]]; then + echo "A prebuilt builder image must be pinned by digest: $builder_image" >&2 + exit 1 +fi + +common_args=( + --rid "$rid" + --target-triple "$target_triple" + --rust-target-triple "$rust_target_triple" + --sysroot "$sysroot" + --builder-identity "$builder_identity" + --glibc-baseline "$max_glibc" + --package-version "$package_version" + --partition-alloc + --upstream-v8 + --output /workspace/artifacts/nuget-packages +) +case "$rid" in + linux-x64) v8_cpu=x64 ;; + linux-arm64) v8_cpu=arm64 ;; +esac +v8_root_host="$repo_root/artifacts/native-engine-v8/$rid/v8" +if [[ -f "$v8_root_host/out/$v8_cpu/ReleasePartitionAlloc/obj/libv8_monolith.a" ]]; then + common_args+=(--v8-root "/workspace/artifacts/native-engine-v8/$rid/v8") +fi +if [[ "$stage" == build && "$rid" == linux-arm64 ]]; then + common_args+=(--defer-target-execution) +fi + +docker run --rm \ + --platform linux/amd64 \ + --user "$(id -u):$(id -g)" \ + --env HOME=/tmp/webscene-home \ + --env DOTNET_CLI_HOME=/tmp/webscene-home \ + --env CARGO_HOME=/tmp/webscene-home/.cargo \ + --env NUGET_PACKAGES=/tmp/webscene-home/.nuget/packages \ + --env "SOURCE_DATE_EPOCH=$source_date_epoch" \ + --env "CARGO_BUILD_TARGET=$rust_target_triple" \ + --env "$cargo_linker_name=clang" \ + --env "$cargo_rustflags_name=-C link-arg=--target=$target_triple -C link-arg=--sysroot=$sysroot --remap-path-prefix=/workspace=." \ + "${docker_mount_args[@]}" \ + --workdir /workspace \ + "$builder_image" \ + scripts/build-native-engine-runtime.sh "${common_args[@]}" + +native_path="$(find "$repo_root/artifacts/native-engine-runtime-build" -path "*/$rid*/libwebscene_native_engine.so" -print -quit)" +if [[ -z "$native_path" ]]; then + echo "Unable to locate the $rid native library for ABI verification." >&2 + exit 1 +fi +python3 "$repo_root/scripts/verify-linux-native-abi.py" "$native_path" \ + --rid "$rid" \ + --max-glibc "$max_glibc" \ + --max-glibcxx "$max_glibcxx" \ + --max-cxxabi "$max_cxxabi" \ + --output "${native_path%/*}/$rid-abi.json" diff --git a/scripts/build-native-engine-runtime-linux-container.sh b/scripts/build-native-engine-runtime-linux-container.sh deleted file mode 100755 index 53389f402..000000000 --- a/scripts/build-native-engine-runtime-linux-container.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env bash -set -uo pipefail - -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -v8_root="$repo_root/artifacts/native-engine-v8/linux-x64/v8" -html_parser=html5ever -css_parser=cssparser -selector_parser=servo -dom_bindings=generated -v8_snapshot=bootstrap -cmake_build_type=Release -thin_lto=false -disable_wasm=false -partition_alloc=false -graphics_sdk= -arguments=("$@") -for ((index = 0; index < ${#arguments[@]}; ++index)); do - argument="${arguments[index]}" - case "$argument" in - --v8-root) v8_root="${arguments[++index]:-}" ;; - --html-parser) html_parser="${arguments[++index]:-}" ;; - --css-parser) css_parser="${arguments[++index]:-}" ;; - --selector-parser) selector_parser="${arguments[++index]:-}" ;; - --dom-bindings) dom_bindings="${arguments[++index]:-}" ;; - --v8-snapshot) v8_snapshot="${arguments[++index]:-}" ;; - --cmake-build-type) cmake_build_type="${arguments[++index]:-}" ;; - --graphics-sdk) graphics_sdk="${arguments[++index]:-}" ;; - --thin-lto) thin_lto=true ;; - --disable-wasm) disable_wasm=true ;; - --partition-alloc) partition_alloc=true ;; - esac -done -build_variant="-$html_parser-$css_parser-$selector_parser-$dom_bindings-$v8_snapshot" -v8_configuration=Release -if [[ -n "$graphics_sdk" ]]; then - build_variant+=-graphics -fi -if [[ "$cmake_build_type" == RelWithDebInfo ]]; then - build_variant+=-symbols -fi -if [[ "$thin_lto" == true ]]; then - build_variant+=-thinlto-llvm - v8_configuration=ReleaseThinLto -fi -if [[ "$disable_wasm" == true ]]; then - build_variant+=-no-wasm - v8_configuration+=NoWasm -fi -if [[ "$partition_alloc" == true ]]; then - build_variant+=-partitionalloc - v8_configuration+=PartitionAlloc -fi -build_variant+=-inspector -build_dir="$repo_root/artifacts/native-engine-runtime-build/linux-x64$build_variant" - -set +e -"$repo_root/scripts/build-native-engine-runtime.sh" "$@" -package_status=$? - -native_test_status=0 -icu_data="$v8_root/out/x64/$v8_configuration/icudtl.dat" -if [[ -d "$build_dir" ]]; then - if [[ -f "$icu_data" ]]; then - cmake -E copy_if_different "$icu_data" "$build_dir/icudtl.dat" - fi - ctest --test-dir "$build_dir" -C Release --output-on-failure - native_test_status=$? - - if ((native_test_status != 0)) && [[ -x "$build_dir/webscene_native_engine_tests" ]]; then - gdb \ - --batch \ - -ex "set pagination off" \ - -ex run \ - -ex "thread apply all bt" \ - --args "$build_dir/webscene_native_engine_tests" || true - fi -fi -set -e - -if ((package_status != 0)); then - exit "$package_status" -fi -exit "$native_test_status" diff --git a/scripts/build-native-engine-runtime.ps1 b/scripts/build-native-engine-runtime.ps1 index 3b488e2ff..59fd7d287 100644 --- a/scripts/build-native-engine-runtime.ps1 +++ b/scripts/build-native-engine-runtime.ps1 @@ -363,7 +363,7 @@ $env:WEBSCENE_VARIABLE_FONT_INSTANCING = '1' try { & dotnet run ` --project (Join-Path $repoRoot "tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj") ` - -c Release -- ` + -c Release -f net10.0 -- ` --selection required ` --native-library $packageNativePath ` --native-cache-directory (Join-Path $buildDir "code-cache") ` diff --git a/scripts/build-native-engine-runtime.sh b/scripts/build-native-engine-runtime.sh index 772ffe701..69c775837 100755 --- a/scripts/build-native-engine-runtime.sh +++ b/scripts/build-native-engine-runtime.sh @@ -22,9 +22,17 @@ disable_wasm=false partition_alloc=false graphics_sdk= cmake_build_type=Release +target_triple= +rust_target_triple= +sysroot= +builder_identity= +glibc_baseline= +depot_tools_commit=ca054941f756b50e1a3d83727270d879bec1f331 +defer_target_execution=false +finalize_only=false usage() { - echo "Usage: $0 --rid osx-arm64|osx-x64|linux-arm64|linux-x64 [--output DIR] [--package-version VERSION] [--v8-root DIR] [--v8-output-root DIR] [--v8-workspace DIR] [--v8-sdk-output DIR] [--v8-revision REVISION] [--html-parser legacy|html5ever] [--css-parser legacy|cssparser] [--selector-parser legacy|servo] [--dom-bindings legacy|generated] [--v8-snapshot none|bootstrap] [--cmake-build-type Release|RelWithDebInfo] [--upstream-v8] [--thin-lto] [--disable-wasm] [--partition-alloc] [--graphics-sdk DIR]" >&2 + echo "Usage: $0 --rid osx-arm64|osx-x64|linux-arm64|linux-x64 [--output DIR] [--package-version VERSION] [--v8-root DIR] [--v8-output-root DIR] [--v8-workspace DIR] [--v8-sdk-output DIR] [--v8-revision REVISION] [--html-parser legacy|html5ever] [--css-parser legacy|cssparser] [--selector-parser legacy|servo] [--dom-bindings legacy|generated] [--v8-snapshot none|bootstrap] [--cmake-build-type Release|RelWithDebInfo] [--upstream-v8] [--thin-lto] [--disable-wasm] [--partition-alloc] [--graphics-sdk DIR] [--target-triple TRIPLE] [--rust-target-triple TRIPLE] [--sysroot DIR] [--builder-identity ID] [--glibc-baseline VERSION] [--depot-tools-commit SHA] [--defer-target-execution|--finalize-only]" >&2 } while (($# > 0)); do @@ -37,6 +45,14 @@ while (($# > 0)); do --v8-workspace) v8_workspace="${2:-}"; shift 2 ;; --v8-sdk-output) v8_sdk_output="${2:-}"; shift 2 ;; --v8-revision) v8_revision="${2:-}"; shift 2 ;; + --target-triple) target_triple="${2:-}"; shift 2 ;; + --rust-target-triple) rust_target_triple="${2:-}"; shift 2 ;; + --sysroot) sysroot="${2:-}"; shift 2 ;; + --builder-identity) builder_identity="${2:-}"; shift 2 ;; + --glibc-baseline) glibc_baseline="${2:-}"; shift 2 ;; + --depot-tools-commit) depot_tools_commit="${2:-}"; shift 2 ;; + --defer-target-execution) defer_target_execution=true; shift ;; + --finalize-only) finalize_only=true; shift ;; --html-parser) html_parser="${2:-}"; shift 2 ;; --css-parser) css_parser="${2:-}"; shift 2 ;; --selector-parser) selector_parser="${2:-}"; shift 2 ;; @@ -134,10 +150,81 @@ if [[ -z "$package_version" && "${WEBSCENE_NATIVE_V8_ONLY:-0}" != 1 ]]; then exit 1 fi -if [[ "$(uname -s)" != "$expected_kernel" || "$(uname -m)" != "$expected_machine" ]]; then - echo "RID '$rid' must be built natively on $expected_kernel/$expected_machine; current host is $(uname -s)/$(uname -m)." >&2 +macos_arm64_to_x64=false +host_kernel="$(uname -s)" +host_machine="$(uname -m)" +if [[ "$rid" == osx-x64 && "$host_kernel" == Darwin && "$host_machine" == arm64 ]]; then + macos_arm64_to_x64=true +fi +if [[ "$expected_kernel" == Darwin \ + && ( "$host_kernel" != "$expected_kernel" \ + || ( "$host_machine" != "$expected_machine" && "$macos_arm64_to_x64" != true ) ) ]]; then + echo "RID '$rid' must be built natively on $expected_kernel/$expected_machine; current host is $host_kernel/$host_machine." >&2 exit 1 fi +if [[ "$expected_kernel" == Darwin && -z "$rust_target_triple" ]]; then + if [[ "$cpu" == x64 ]]; then + rust_target_triple=x86_64-apple-darwin + else + rust_target_triple=aarch64-apple-darwin + fi +fi +if [[ "$expected_kernel" == Darwin ]]; then + rust_version=1.90.0 + rust_mac_arm64_sha256=9772d20d5cd736079a0ee84d00e6697cf2084f0fc4621b011e24e6f2d08d2d7f + rust_mac_x64_std_sha256=dd731e6f9f30cb9b2928b92b084d2f12a3abf06a481ecbd8c3553c3e6f742139 + rust_prefix="${RUNNER_TOOL_CACHE:-${RUNNER_TEMP:-$repo_root/artifacts/toolchains}}/webscene-rust-$rust_version" + rust_complete="$rust_prefix/.webscene-complete" + if [[ ! -f "$rust_complete" ]]; then + rust_download_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/webscene-rust.XXXXXX")" + ( + cd "$rust_download_dir" + host_archive="rust-$rust_version-aarch64-apple-darwin.tar.xz" + x64_std_archive="rust-std-$rust_version-x86_64-apple-darwin.tar.xz" + curl --fail --silent --show-error --location \ + --retry 5 --retry-delay 2 --retry-all-errors --connect-timeout 20 \ + --remote-name "https://static.rust-lang.org/dist/$host_archive" + echo "$rust_mac_arm64_sha256 $host_archive" | shasum -a 256 -c - + tar -xf "$host_archive" + "${host_archive%.tar.xz}/install.sh" --prefix="$rust_prefix" --without=rust-docs + curl --fail --silent --show-error --location \ + --retry 5 --retry-delay 2 --retry-all-errors --connect-timeout 20 \ + --remote-name "https://static.rust-lang.org/dist/$x64_std_archive" + echo "$rust_mac_x64_std_sha256 $x64_std_archive" | shasum -a 256 -c - + tar -xf "$x64_std_archive" + "${x64_std_archive%.tar.xz}/install.sh" --prefix="$rust_prefix" + : > "$rust_complete" + ) + fi + export PATH="$rust_prefix/bin:$PATH" + if [[ "$(rustc --version)" != "rustc $rust_version "* ]]; then + echo "Pinned macOS Rust toolchain validation failed: $(rustc --version)" >&2 + exit 1 + fi +fi +if [[ "$expected_kernel" == Linux ]]; then + case "$rid:$target_triple" in + linux-x64:x86_64-linux-gnu|linux-arm64:aarch64-linux-gnu) ;; + *) echo "RID '$rid' requires its locked Linux target triple, not '$target_triple'." >&2; exit 1 ;; + esac + case "$rid:$rust_target_triple" in + linux-x64:x86_64-unknown-linux-gnu|linux-arm64:aarch64-unknown-linux-gnu) ;; + *) echo "RID '$rid' requires its locked Rust target triple, not '$rust_target_triple'." >&2; exit 1 ;; + esac + if [[ "$finalize_only" == false && ! -d "$sysroot" ]]; then + echo "Linux cross-build sysroot is missing: $sysroot" >&2 + exit 1 + fi + if [[ -z "$builder_identity" || -z "$glibc_baseline" ]]; then + echo "Linux release builds require --builder-identity and --glibc-baseline." >&2 + exit 1 + fi +fi + +if [[ "$finalize_only" == true && -z "$v8_root" ]]; then + v8_workspace="${v8_workspace:-$repo_root/artifacts/native-engine-v8/$rid}" + v8_root="$v8_workspace/v8" +fi if [[ -z "$v8_root" ]]; then v8_workspace="${v8_workspace:-$repo_root/artifacts/native-engine-v8/$rid}" @@ -145,7 +232,10 @@ if [[ -z "$v8_root" ]]; then v8_root="$v8_workspace/v8" mkdir -p "$v8_workspace" - if [[ ! -d "$depot_tools/.git" ]]; then + if [[ ! -d "$depot_tools/.git" && -d /opt/depot_tools/.git ]]; then + git clone --no-checkout /opt/depot_tools "$depot_tools" + git -C "$depot_tools" checkout --detach "$depot_tools_commit" + elif [[ ! -d "$depot_tools/.git" ]]; then clone_attempt=1 while ! git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git "$depot_tools"; do if ((clone_attempt >= 3)); then @@ -157,6 +247,10 @@ if [[ -z "$v8_root" ]]; then clone_attempt=$((clone_attempt + 1)) done fi + if [[ "$(git -C "$depot_tools" rev-parse HEAD)" != "$depot_tools_commit" ]]; then + git -C "$depot_tools" fetch origin "$depot_tools_commit" + git -C "$depot_tools" checkout --detach "$depot_tools_commit" + fi export PATH="$depot_tools:$PATH" if [[ ! -f "$depot_tools/python3_bin_reldir.txt" ]]; then "$depot_tools/ensure_bootstrap" @@ -210,17 +304,26 @@ if [[ -z "$v8_root" ]]; then "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocMacVisibilityPatch.txt" fi if [[ "$expected_kernel" == Linux ]]; then + webscene_apply_patch_once "$v8_root/buildtools" "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8LibcxxMemoryResourcePatch.txt" webscene_apply_patch_once "$v8_root/build" "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8BuildNoCrelPatch.txt" + if [[ "$cpu" == arm64 ]]; then + webscene_apply_patch_once "$v8_root/third_party/partition_alloc/src" "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocGlibc227Arm64Patch.txt" + fi fi - gn_args="chrome_pgo_phase=0 fatal_linker_warnings=false is_cfi=false is_component_build=false is_debug=false symbol_level=0 target_cpu=\"$cpu\" treat_warnings_as_errors=false use_clang_modules=false use_custom_libcxx=false use_thin_lto=$thin_lto v8_embedder_string=\"-WebScene\" v8_enable_fuzztest=false v8_enable_partition_alloc=$partition_alloc v8_enable_pointer_compression=true v8_enable_pointer_compression_shared_cage=true v8_enable_sandbox=false v8_enable_static_roots=false v8_enable_31bit_smis_on_64bit_arch=false v8_enable_temporal_support=false v8_enable_webassembly=$v8_webassembly v8_monolithic=true v8_use_external_startup_data=false v8_target_cpu=\"$cpu\"" + gn_args="chrome_pgo_phase=0 fatal_linker_warnings=false is_cfi=false is_component_build=false is_debug=false symbol_level=0 target_cpu=\"$cpu\" treat_warnings_as_errors=false use_clang_modules=false use_thin_lto=$thin_lto v8_embedder_string=\"-WebScene\" v8_enable_fuzztest=false v8_enable_partition_alloc=$partition_alloc v8_enable_pointer_compression=true v8_enable_pointer_compression_shared_cage=true v8_enable_sandbox=false v8_enable_static_roots=false v8_enable_31bit_smis_on_64bit_arch=false v8_enable_temporal_support=false v8_enable_webassembly=$v8_webassembly v8_monolithic=true v8_use_external_startup_data=false v8_target_cpu=\"$cpu\"" if [[ "$expected_kernel" == Linux ]]; then - # V8 15.3 requires C++20 library headers that are newer than its downloaded - # Debian Bullseye sysroot. Build inside the pinned Ubuntu 22.04 image - # against that image's libstdc++ and glibc 2.35 instead. + # V8 15.3 requires C++20 library headers that are newer than the glibc 2.27 + # target sysroot provides. Use Chromium's bundled libc++ while retaining + # the locked old-glibc sysroot for the platform ABI. # Keep V8's bundled LLD for its host tools; the reviewed build patch above # disables only CREL emission so Jammy can consume the archive. - gn_args+=" use_lld=true use_sysroot=false v8_monolithic_for_shared_library=true" + gn_args+=" use_custom_libcxx=true use_lld=true use_sysroot=true target_sysroot=\"$sysroot\" use_glib=false v8_monolithic_for_shared_library=true" + elif [[ "$expected_kernel" == Darwin ]]; then + # WebScene's embedding targets use the libc++ supplied by the selected + # macOS SDK. Build V8 against the same ABI; Chromium's bundled libc++ uses + # the std::__Cr namespace and cannot be linked with Apple's system libc++. + gn_args+=" use_custom_libcxx=false" fi if [[ "$partition_alloc" == true \ && ( "$expected_kernel" == Linux || "$expected_kernel" == Darwin ) ]]; then @@ -233,7 +336,64 @@ if [[ -z "$v8_root" ]]; then ( cd "$v8_root" gn gen "out/$cpu/$v8_configuration" --args="$gn_args" - ninja -C "out/$cpu/$v8_configuration" obj/libv8_monolith.a + if [[ "$expected_kernel" == Linux && "$cpu" == arm64 ]]; then + partition_alloc_buildflags_relative="gen/third_party/partition_alloc/src/partition_alloc/buildflags.h" + partition_alloc_buildflags="out/$cpu/$v8_configuration/$partition_alloc_buildflags_relative" + ninja -C "out/$cpu/$v8_configuration" "$partition_alloc_buildflags_relative" + if [[ ! -f "$partition_alloc_buildflags" ]]; then + echo "PartitionAlloc build flags were not generated at '$partition_alloc_buildflags'." >&2 + exit 1 + fi + # V8's embedder overrides can retain ARM MTE even when the standalone + # PartitionAlloc default is patched. glibc 2.27 has no sys/ifunc.h, so + # force the generated target flag off before Ninja consumes it. + sed -i \ + 's/^#define PA_BUILDFLAG_INTERNAL_HAS_MEMORY_TAGGING() (1)$/#define PA_BUILDFLAG_INTERNAL_HAS_MEMORY_TAGGING() (0)/' \ + "$partition_alloc_buildflags" + if ! grep -Fqx '#define PA_BUILDFLAG_INTERNAL_HAS_MEMORY_TAGGING() (0)' "$partition_alloc_buildflags"; then + echo "Unable to disable PartitionAlloc memory tagging for the glibc 2.27 ARM64 target." >&2 + exit 1 + fi + fi + v8_ninja_targets=(obj/libv8_monolith.a) + if [[ "$expected_kernel" == Linux ]]; then + # Cross builds need target-architecture C++ runtime archives in the + # primary toolchain. V8's ARM64 monolith otherwise builds libc++ only for + # the x64 host-tools toolchain used by mksnapshot. + v8_ninja_targets+=( + obj/buildtools/third_party/libc++/libc++.a + obj/buildtools/third_party/libc++abi/libc++abi.a) + fi + ninja -C "out/$cpu/$v8_configuration" "${v8_ninja_targets[@]}" + if [[ "$expected_kernel" == Linux ]]; then + # Chromium emits thin archives here. They only contain paths to the + # adjacent object files, so restoring just the archives from the V8 SDK + # cache makes the final WebScene link fail. Repack every member into a + # regular deterministic archive before the cache is populated. + llvm_ar="$v8_root/third_party/llvm-build/Release+Asserts/bin/llvm-ar" + for archive in \ + "out/$cpu/$v8_configuration/obj/buildtools/third_party/libc++/libc++.a" \ + "out/$cpu/$v8_configuration/obj/buildtools/third_party/libc++abi/libc++abi.a"; do + archive_dir="$(dirname "$archive")" + archive_name="$(basename "$archive")" + regular_archive="$archive_name.regular.$$" + ( + cd "$archive_dir" + mapfile -t archive_members < <("$llvm_ar" t "$archive_name") + if (( ${#archive_members[@]} == 0 )); then + echo "V8 C++ runtime archive has no members: $archive" >&2 + exit 1 + fi + rm -f "$regular_archive" + "$llvm_ar" rcD "$regular_archive" "${archive_members[@]}" + if [[ "$(head -c 7 "$regular_archive")" != '!' ]]; then + echo "Failed to materialize regular V8 C++ runtime archive: $archive" >&2 + exit 1 + fi + mv "$regular_archive" "$archive_name" + ) + done + fi ) v8_output_root="$v8_root/out/$cpu/$v8_configuration" fi @@ -298,11 +458,27 @@ if [[ "$expected_kernel" == Linux ]] \ echo "The V8 SDK at '$v8_root' was not built with the required patched LLD configuration." >&2 exit 1 fi +if [[ "$expected_kernel" == Linux ]] \ + && { ! grep -Eq '^use_sysroot *= *true$' "$v8_args" \ + || ! grep -Fq "target_sysroot = \"$sysroot\"" "$v8_args"; }; then + echo "The V8 SDK at '$v8_root' was not built against the locked target sysroot." >&2 + exit 1 +fi if [[ "$expected_kernel" == Linux ]] \ && ! grep -Eq '^v8_monolithic_for_shared_library *= *true$' "$v8_args"; then echo "The V8 SDK at '$v8_root' is not safe to link into a shared library." >&2 exit 1 fi +if [[ "$expected_kernel" == Linux ]] \ + && ! grep -Eq '^use_custom_libcxx *= *true$' "$v8_args"; then + echo "The V8 SDK at '$v8_root' was not built with Chromium's required Linux libc++." >&2 + exit 1 +fi +if [[ "$expected_kernel" == Darwin ]] \ + && ! grep -Eq '^use_custom_libcxx *= *false$' "$v8_args"; then + echo "The V8 SDK at '$v8_root' was not built with the macOS system libc++." >&2 + exit 1 +fi if [[ "$partition_alloc" == true \ && ( "$expected_kernel" == Linux || "$expected_kernel" == Darwin ) ]] \ && { ! grep -Eq '^use_allocator_shim *= *false$' "$v8_args" \ @@ -347,10 +523,19 @@ cmake_args=( -DWEBSCENE_NATIVE_ENGINE_V8_SNAPSHOT="$v8_snapshot" -DWEBSCENE_V8_ROOT="$v8_root" -DWEBSCENE_V8_OUTPUT_ROOT="$v8_output_root" + -DWEBSCENE_NATIVE_ENGINE_DEFER_TARGET_EXECUTION="$defer_target_execution" ) -macos_deployment_target=12.0 +macos_deployment_target=14.0 if [[ "$expected_kernel" == Darwin ]]; then - cmake_args+=(-DCMAKE_OSX_DEPLOYMENT_TARGET="$macos_deployment_target") + macos_architecture=arm64 + if [[ "$cpu" == x64 ]]; then + macos_architecture=x86_64 + fi + cmake_args+=( + -DCMAKE_OSX_ARCHITECTURES="$macos_architecture" + -DCMAKE_OSX_DEPLOYMENT_TARGET="$macos_deployment_target" + -DWEBSCENE_RUST_TARGET_TRIPLE="$rust_target_triple" + ) fi if [[ "$thin_lto" == true ]]; then v8_llvm_bin="$v8_root/third_party/llvm-build/Release+Asserts/bin" @@ -384,43 +569,132 @@ if [[ "$thin_lto" == true ]]; then -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=lld -DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=lld ) -elif [[ "$expected_kernel" == Linux ]]; then - # V8's Linux archive must be linked with LLD. The compiler is selectable so - # the Ubuntu 22.04 compatibility image can use GCC 11's complete C++20 - # standard library instead of Jammy's Clang 14 source_location support. - linux_cxx="${CXX:-clang++}" - if ! command -v "$linux_cxx" >/dev/null 2>&1 || ! command -v ld.lld >/dev/null 2>&1; then - echo "Linux native runtime builds require '$linux_cxx' and ld.lld." >&2 +elif [[ "$expected_kernel" == Linux && "$finalize_only" == false ]]; then + # Compile the embedding library with the exact Chromium LLVM and libc++ + # revision used for V8. New libc++ headers can require compiler features and + # configuration defines absent from the builder image's host toolchain. + target_library_dir="$sysroot/usr/lib/$target_triple" + target_include_dir="$sysroot/usr/include" + v8_libcxx_config_include="$v8_root/buildtools/third_party/libc++" + v8_libcxx_include="$v8_root/third_party/libc++/src/include" + v8_libcxxabi_include="$v8_root/third_party/libc++abi/src/include" + v8_libcxx_archive="$v8_output_root/obj/buildtools/third_party/libc++/libc++.a" + v8_libcxxabi_archive="$v8_output_root/obj/buildtools/third_party/libc++abi/libc++abi.a" + v8_llvm_root="$v8_root/third_party/llvm-build/Release+Asserts" + v8_llvm_bin="$v8_llvm_root/bin" + for target_dependency in \ + "$target_include_dir/openssl/ssl.h" \ + "$target_library_dir/libcrypto.a" \ + "$target_library_dir/libssl.a" \ + "$target_include_dir/zlib.h" \ + "$target_library_dir/libz.a" \ + "$v8_libcxx_config_include/__config_site" \ + "$v8_libcxx_config_include/__assertion_handler" \ + "$v8_libcxx_include/source_location" \ + "$v8_libcxxabi_include/cxxabi.h" \ + "$v8_libcxx_archive" \ + "$v8_libcxxabi_archive" \ + "$v8_llvm_bin/clang" \ + "$v8_llvm_bin/clang++" \ + "$v8_llvm_bin/llvm-ar" \ + "$v8_llvm_bin/ld.lld"; do + if [[ ! -e "$target_dependency" ]]; then + echo "Linux sysroot is missing required native dependency '$target_dependency'." >&2 + exit 1 + fi + done + for runtime_archive in "$v8_libcxx_archive" "$v8_libcxxabi_archive"; do + if [[ "$(head -c 7 "$runtime_archive")" != '!' ]]; then + echo "Linux V8 C++ runtime dependency is not a self-contained regular archive: '$runtime_archive'." >&2 + exit 1 + fi + done + if ! "$v8_llvm_bin/llvm-ar" t "$v8_libcxx_archive" \ + | grep -Eq '(^|/)memory_resource\.o$'; then + echo "Linux V8 libc++ archive does not provide std::pmr support: '$v8_libcxx_archive'." >&2 exit 1 fi + v8_llvm_ranlib="$v8_llvm_bin/llvm-ranlib" + if [[ ! -x "$v8_llvm_ranlib" ]]; then + ln -s "$v8_llvm_bin/llvm-ar" "$v8_llvm_ranlib" + fi cmake_args+=( - -DCMAKE_CXX_COMPILER="$linux_cxx" + -DCMAKE_TOOLCHAIN_FILE="$repo_root/scripts/linux-glibc-toolchain.cmake" + -DCMAKE_SYSROOT="$sysroot" + -DCMAKE_C_COMPILER="$v8_llvm_bin/clang" + -DCMAKE_CXX_COMPILER="$v8_llvm_bin/clang++" + -DCMAKE_AR="$v8_llvm_bin/llvm-ar" + -DCMAKE_RANLIB="$v8_llvm_ranlib" + -DCMAKE_C_COMPILER_AR="$v8_llvm_bin/llvm-ar" + -DCMAKE_C_COMPILER_RANLIB="$v8_llvm_ranlib" + -DCMAKE_CXX_COMPILER_AR="$v8_llvm_bin/llvm-ar" + -DCMAKE_CXX_COMPILER_RANLIB="$v8_llvm_ranlib" + -DCMAKE_LINKER="$v8_llvm_bin/ld.lld" + -DWEBSCENE_LINUX_TARGET_TRIPLE="$target_triple" + -DWEBSCENE_RUST_TARGET_TRIPLE="$rust_target_triple" + -DOPENSSL_ROOT_DIR="$sysroot/usr" + -DOPENSSL_INCLUDE_DIR="$target_include_dir" + -DOPENSSL_CRYPTO_LIBRARY="$target_library_dir/libcrypto.a" + -DOPENSSL_SSL_LIBRARY="$target_library_dir/libssl.a" + -DZLIB_INCLUDE_DIR="$target_include_dir" + -DZLIB_LIBRARY="$target_library_dir/libz.a" + -DCMAKE_SKIP_RPATH=TRUE + "-DCMAKE_C_FLAGS=-ffile-prefix-map=$repo_root=. -fdebug-prefix-map=$repo_root=." + "-DCMAKE_CXX_FLAGS=-ffile-prefix-map=$repo_root=. -fdebug-prefix-map=$repo_root=. -nostdinc++ -nostdlib++ -I$v8_libcxx_config_include -isystem$v8_libcxx_include -isystem$v8_libcxxabi_include -include new -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS -D_LIBCXXABI_DISABLE_VISIBILITY_ANNOTATIONS -D_LIBCPP_INSTRUMENTED_WITH_ASAN=0 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_EXTENSIVE" + "-DCMAKE_CXX_STANDARD_LIBRARIES=$v8_libcxx_archive $v8_libcxxabi_archive -pthread" -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=lld - -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=lld + "-DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=lld -Wl,--build-id=sha1" ) fi if [[ "$expected_kernel" == Darwin && "$cmake_build_type" == Release ]]; then - # Keep line tables only until dsymutil has emitted the exact shipped - # binary's external symbols. strip removes them from the runtime before - # packaging, so diagnostics do not increase the installed footprint. cmake_args+=("-DCMAKE_CXX_FLAGS_RELEASE=-O3 -DNDEBUG -gline-tables-only") fi -cmake "${cmake_args[@]}" -cmake --build "$build_dir" --config "$cmake_build_type" --parallel -cmake -E copy_if_different "$icu_data" "$build_dir/icudtl.dat" -ctest_args=(--test-dir "$build_dir" -C "$cmake_build_type" --output-on-failure) -# Hosted package builders prove linkage and CPU contracts; real GPU execution -# remains mandatory on the explicitly enrolled hardware qualification runners. -if [[ "${WEBSCENE_NATIVE_SKIP_HARDWARE_TESTS:-0}" == 1 ]]; then - ctest_args+=(-LE hardware) +if [[ "$finalize_only" == false ]]; then + cmake "${cmake_args[@]}" + cmake --build "$build_dir" --config "$cmake_build_type" --parallel + cmake -E copy_if_different "$icu_data" "$build_dir/icudtl.dat" +fi + +if [[ "$finalize_only" == true ]]; then + snapshot_builder="$build_dir/webscene_v8_snapshot_builder" + if [[ ! -x "$snapshot_builder" ]]; then + echo "Cross-build output is missing its target snapshot builder: $snapshot_builder" >&2 + exit 1 + fi + "$snapshot_builder" \ + "$icu_data" \ + "$build_dir/webscene_v8_bootstrap.js" \ + "$build_dir/webscene_bootstrap_snapshot.bin" \ + "$build_dir/webscene_bootstrap_snapshot.meta" +fi +if [[ "$defer_target_execution" == false || "$finalize_only" == true ]]; then + if [[ "$expected_kernel" == Linux ]]; then + # Production DSOs intentionally contain no RPATH. Give native test + # executables an explicit, process-local route to the just-built DSO. + test_library_path="$build_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + ctest_args=(--test-dir "$build_dir" -C "$cmake_build_type" --output-on-failure) + if [[ "${WEBSCENE_NATIVE_SKIP_HARDWARE_TESTS:-0}" == 1 ]]; then + ctest_args+=(-LE hardware) + fi + cmake -E env "LD_LIBRARY_PATH=$test_library_path" ctest "${ctest_args[@]}" + else + ctest_args=(--test-dir "$build_dir" -C "$cmake_build_type" --output-on-failure) + if [[ "${WEBSCENE_NATIVE_SKIP_HARDWARE_TESTS:-0}" == 1 ]]; then + ctest_args+=(-LE hardware) + fi + ctest "${ctest_args[@]}" + fi fi -ctest "${ctest_args[@]}" native_path="$build_dir/$native_name" if [[ ! -f "$native_path" ]]; then echo "Native engine build did not produce '$native_path'." >&2 exit 1 fi +if [[ "$defer_target_execution" == true && "$finalize_only" == false ]]; then + echo "Cross-build staged for native finalization: $build_dir" + exit 0 +fi if [[ "$expected_kernel" == Darwin ]]; then actual_macos_deployment_target="$( xcrun vtool -show-build "$native_path" | @@ -430,6 +704,7 @@ if [[ "$expected_kernel" == Darwin ]]; then echo "Native engine deployment target is '$actual_macos_deployment_target'; expected '$macos_deployment_target'." >&2 exit 1 fi + native_dsym_path="$native_path.dSYM" cmake -E remove_directory "$native_dsym_path" dsymutil "$native_path" -o "$native_dsym_path" @@ -504,6 +779,9 @@ pack_args=( "-p:WebSceneNativeEngineDomBindings=$dom_bindings" "-p:WebSceneNativeEngineV8Snapshot=$v8_snapshot" "-p:WebSceneNativeEngineConfiguration=$cmake_build_type" + "-p:WebSceneNativeEngineBuilderIdentity=$builder_identity" + "-p:WebSceneNativeEngineTargetTriple=$target_triple" + "-p:WebSceneNativeEngineGlibcBaseline=$glibc_baseline" ) if [[ -n "$graphics_sdk" ]]; then graphics_stage_root="$(mktemp -d "$build_dir/graphics-package.XXXXXX")" @@ -521,7 +799,14 @@ if [[ "$html_parser" == html5ever ]]; then "-p:WebSceneNativeEngineHtmlParserNoticesPath=$repo_root/experiments/WebScene.NativeEngine.Probe/native/html_parser/THIRD-PARTY-NOTICES.md") fi pack_args+=("-p:PackageVersion=$package_version") -dotnet pack "${pack_args[@]}" +# Self-hosted runners retain .NET build-server processes between invocations and +# jobs. This is especially problematic when an Apple Silicon runner alternates +# between native arm64 and Rosetta x64 SDKs: a later command can wait forever on +# a server from the other architecture. Ensure validation is isolated from any +# persistent server state and do not create new reusable servers below. +dotnet build-server shutdown + +dotnet pack "${pack_args[@]}" --disable-build-servers package_path="$output_dir/WebScene.NativeEngine.Runtime.$rid.$package_version.nupkg" if [[ ! -f "$package_path" ]]; then @@ -536,7 +821,7 @@ package_native_path="$package_smoke_dir/runtimes/$rid/native/$native_name" WEBSCENE_VARIABLE_FONT_INSTANCING=1 dotnet run \ --project "$repo_root/tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj" \ - -c Release -- \ + -c Release -f net10.0 --disable-build-servers -- \ --selection required \ --native-library "$package_native_path" \ --native-cache-directory "$build_dir/code-cache" \ @@ -555,13 +840,13 @@ done WEBSCENE_TEST_NATIVE_LIBRARY="$package_native_path" \ WEBSCENE_VARIABLE_FONT_INSTANCING=1 \ dotnet test "$repo_root/tests/WebScene.Backend.Avalonia.Tests/WebScene.Backend.Avalonia.Tests.csproj" \ - -c Release -f net10.0 \ + -c Release -f net10.0 --disable-build-servers \ --filter 'FullyQualifiedName~NativeWebFontCacheTests|FullyQualifiedName~VariableWebFontTests|FullyQualifiedName~SvgPictureRenderingTests' WEBSCENE_NATIVE_ENGINE_PATH="$package_native_path" \ dotnet run \ --project "$repo_root/benchmarks/WebScene.NativeEngine.Benchmarks/WebScene.NativeEngine.Benchmarks.csproj" \ - -c Release -- \ + -c Release --disable-build-servers -- \ probe native-interop-race --batches 100 --width 32 consumer_smoke_root="$repo_root/artifacts/native-engine-consumer-smoke" diff --git a/scripts/linux-glibc-toolchain.cmake b/scripts/linux-glibc-toolchain.cmake new file mode 100644 index 000000000..6a204939a --- /dev/null +++ b/scripts/linux-glibc-toolchain.cmake @@ -0,0 +1,49 @@ +set(CMAKE_SYSTEM_NAME Linux) + +# CMake re-evaluates this toolchain inside try_compile projects. Explicitly +# forward WebScene's target identity so compiler ABI checks remain cross builds. +set(CMAKE_TRY_COMPILE_PLATFORM_VARIABLES + WEBSCENE_LINUX_TARGET_TRIPLE + WEBSCENE_RUST_TARGET_TRIPLE + CMAKE_SYSROOT) + +if(NOT DEFINED WEBSCENE_LINUX_TARGET_TRIPLE) + message(FATAL_ERROR "WEBSCENE_LINUX_TARGET_TRIPLE is required") +endif() +if(NOT DEFINED CMAKE_SYSROOT OR CMAKE_SYSROOT STREQUAL "") + message(FATAL_ERROR "CMAKE_SYSROOT is required") +endif() + +if(WEBSCENE_LINUX_TARGET_TRIPLE STREQUAL "x86_64-linux-gnu") + set(CMAKE_SYSTEM_PROCESSOR x86_64) +elseif(WEBSCENE_LINUX_TARGET_TRIPLE STREQUAL "aarch64-linux-gnu") + set(CMAKE_SYSTEM_PROCESSOR aarch64) +else() + message(FATAL_ERROR "Unsupported Linux target triple: ${WEBSCENE_LINUX_TARGET_TRIPLE}") +endif() + +# The pinned sysroots use Debian multiarch directories. CMake does not always +# infer these while cross-compiling, so make the target layout available to all +# find_package/find_library calls instead of resolving libraries from the host. +set(CMAKE_LIBRARY_ARCHITECTURE "${WEBSCENE_LINUX_TARGET_TRIPLE}") +list(APPEND CMAKE_SYSTEM_LIBRARY_PATH + "/lib/${WEBSCENE_LINUX_TARGET_TRIPLE}" + "/usr/lib/${WEBSCENE_LINUX_TARGET_TRIPLE}") +list(APPEND CMAKE_SYSTEM_INCLUDE_PATH + "/usr/include/${WEBSCENE_LINUX_TARGET_TRIPLE}" + "/usr/include") + +if(NOT DEFINED CMAKE_C_COMPILER) + set(CMAKE_C_COMPILER clang) +endif() +if(NOT DEFINED CMAKE_CXX_COMPILER) + set(CMAKE_CXX_COMPILER clang++) +endif() +set(CMAKE_C_COMPILER_TARGET "${WEBSCENE_LINUX_TARGET_TRIPLE}") +set(CMAKE_CXX_COMPILER_TARGET "${WEBSCENE_LINUX_TARGET_TRIPLE}") +set(CMAKE_FIND_ROOT_PATH "${CMAKE_SYSROOT}") +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) diff --git a/scripts/tests/test_linux_build_policy.py b/scripts/tests/test_linux_build_policy.py new file mode 100644 index 000000000..e99195061 --- /dev/null +++ b/scripts/tests/test_linux_build_policy.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import hashlib +import json +import pathlib +import re +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[2] +PACKAGING = ROOT / "packaging" / "WebScene.NativeEngine.Runtime" + + +class LinuxBuildPolicyTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.lock = json.loads((PACKAGING / "linux-build-lock.json").read_text()) + cls.dockerfile = (PACKAGING / "Dockerfile.linux-glibc").read_text() + cls.finalizer_dockerfile = ( + PACKAGING / "Dockerfile.linux-arm64-finalizer" + ).read_text() + cls.workflow = (ROOT / ".github/workflows/native-runtime-packages.yml").read_text() + cls.build_script = (ROOT / "scripts/build-native-engine-runtime.sh").read_text() + cls.toolchain = (ROOT / "scripts/linux-glibc-toolchain.cmake").read_text() + + def test_all_container_inputs_are_digest_pinned(self) -> None: + from_lines = re.findall( + r"^FROM\s+(\S+)", + self.dockerfile + "\n" + self.finalizer_dockerfile, + re.MULTILINE, + ) + external = [value for value in from_lines if value not in {"x64-sysroot"}] + self.assertTrue(external) + self.assertTrue(all("@sha256:" in value for value in external), external) + self.assertNotIn("apt-get", self.dockerfile) + + def test_lock_and_dockerfile_are_synchronized(self) -> None: + self.assertEqual( + "2.14.2-2.azl3", + self.lock["hostRuntimePackages"]["fontconfig"], + ) + self.assertEqual( + "2.37-3.azl3", + self.lock["hostRuntimePackages"]["dejavu-sans-fonts"], + ) + self.assertEqual( + "2.37-3.azl3", + self.lock["hostRuntimePackages"]["dejavu-sans-mono-fonts"], + ) + self.assertEqual( + "2.37-3.azl3", + self.lock["hostRuntimePackages"]["dejavu-serif-fonts"], + ) + self.assertIn("fontconfig-2.14.2-2.azl3", self.dockerfile) + self.assertIn("dejavu-sans-fonts-2.37-3.azl3", self.dockerfile) + self.assertIn("dejavu-sans-mono-fonts-2.37-3.azl3", self.dockerfile) + self.assertIn("dejavu-serif-fonts-2.37-3.azl3", self.dockerfile) + finalizer = self.lock["arm64Finalizer"] + self.assertIn( + f'{finalizer["image"]}@{finalizer["digest"]}', + self.finalizer_dockerfile, + ) + for package, version in finalizer["packages"].items(): + self.assertIn(f"{package}={version}", self.finalizer_dockerfile) + expected = [self.lock["dotnetSdk"], *self.lock["sysroots"].values()] + for item in expected: + image = item.get("image", item.get("sourceImage")) + self.assertIsNotNone(image) + self.assertIn(f'{image}@{item["digest"]}', self.dockerfile) + toolchain = self.lock["toolchain"] + for value in ( + toolchain["rust"], toolchain["rustArchiveSha256"], + toolchain["rustArm64StdSha256"], toolchain["depotToolsCommit"], + ): + self.assertIn(value, self.dockerfile) + + for patch in self.lock["patches"].values(): + patch_path = PACKAGING / patch["path"] + self.assertTrue(patch_path.is_file(), patch_path) + self.assertEqual( + patch["sha256"], + hashlib.sha256(patch_path.read_bytes()).hexdigest(), + ) + + for key in ("rustMacArm64ArchiveSha256", "rustMacX64StdSha256"): + self.assertIn(toolchain[key], self.build_script) + + def test_release_matrix_contains_both_glibc_rids(self) -> None: + for rid in ("linux-x64", "linux-arm64"): + self.assertIn(f"rid: {rid}", self.workflow) + self.assertIn(f"--expected-rid {rid}", self.workflow) + self.assertIn(f"--native-rid {rid}", self.workflow) + self.assertIn("github.ref_type != 'tag'", self.workflow) + self.assertIn("runs-on: [self-hosted, Linux, X64]", self.workflow) + self.assertIn("--platform linux/arm64", self.workflow) + self.assertNotIn("runs-on: [self-hosted, Linux, ARM64]", self.workflow) + + def test_linux_libcxx_cache_paths_do_not_invalidate_macos_caches(self) -> None: + self.assertEqual(2, self.workflow.count("v8_cache_extra_paths: |")) + self.assertEqual(3, self.workflow.count("v8_cache_extra_paths: ''")) + self.assertEqual(2, self.workflow.count("${{ matrix.v8_cache_extra_paths }}")) + + def test_arm64_disables_memory_tagging_for_glibc_227(self) -> None: + self.assertIn( + "PA_BUILDFLAG_INTERNAL_HAS_MEMORY_TAGGING() (0)", + self.build_script, + ) + self.assertIn("V8PartitionAllocGlibc227Arm64Patch.txt", self.build_script) + + def test_cmake_try_compile_keeps_cross_target_identity(self) -> None: + self.assertIn("CMAKE_TRY_COMPILE_PLATFORM_VARIABLES", self.toolchain) + self.assertIn("WEBSCENE_LINUX_TARGET_TRIPLE", self.toolchain) + self.assertIn("CMAKE_SYSROOT", self.toolchain) + + def test_linux_openssl_is_resolved_only_from_the_target_sysroot(self) -> None: + self.assertIn( + 'elif [[ "$expected_kernel" == Linux && "$finalize_only" == false ]]; then', + self.build_script, + ) + self.assertIn('target_library_dir="$sysroot/usr/lib/$target_triple"', self.build_script) + self.assertIn('-DOPENSSL_CRYPTO_LIBRARY="$target_library_dir/libcrypto.a"', self.build_script) + self.assertIn('-DOPENSSL_SSL_LIBRARY="$target_library_dir/libssl.a"', self.build_script) + + def test_linux_zlib_is_resolved_only_from_the_target_sysroot(self) -> None: + self.assertIn('-DZLIB_INCLUDE_DIR="$target_include_dir"', self.build_script) + self.assertIn('-DZLIB_LIBRARY="$target_library_dir/libz.a"', self.build_script) + self.assertIn('-DCMAKE_SKIP_RPATH=TRUE', self.build_script) + self.assertIn('LD_LIBRARY_PATH=$test_library_path', self.build_script) + self.assertIn( + '-c Release -f net10.0 --', + self.build_script, + ) + self.assertIn( + 'WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../.."', + (ROOT / "experiments/WebScene.NativeEngine.Probe/CMakeLists.txt").read_text(), + ) + + def test_toolchain_exposes_target_multiarch_search_paths(self) -> None: + self.assertIn("CMAKE_LIBRARY_ARCHITECTURE", self.toolchain) + self.assertIn('/usr/lib/${WEBSCENE_LINUX_TARGET_TRIPLE}', self.toolchain) + self.assertIn('/usr/include/${WEBSCENE_LINUX_TARGET_TRIPLE}', self.toolchain) + + def test_linux_runtime_uses_v8_bundled_libcxx(self) -> None: + self.assertIn('v8_root/third_party/libc++/src/include', self.build_script) + self.assertIn('v8_root/third_party/libc++abi/src/include', self.build_script) + self.assertIn('v8_root/buildtools/third_party/libc++', self.build_script) + self.assertIn('__config_site', self.build_script) + self.assertIn('__assertion_handler', self.build_script) + self.assertIn( + 'artifacts/native-engine-v8/linux-*/v8/buildtools/third_party/libc++', + self.workflow, + ) + self.assertIn("-nostdinc++ -nostdlib++", self.build_script) + self.assertIn("CMAKE_CXX_STANDARD_LIBRARIES", self.build_script) + self.assertIn("libc++abi.a", self.build_script) + self.assertIn("third_party/llvm-build/Release+Asserts", self.build_script) + self.assertIn("_LIBCPP_HARDENING_MODE_EXTENSIVE", self.build_script) + self.assertIn("-include new", self.build_script) + self.assertIn('llvm_ar" rcD "$regular_archive"', self.build_script) + self.assertIn("'!'", self.build_script) + self.assertIn("archive_is_regular", self.workflow) + self.assertIn("V8LibcxxMemoryResourcePatch.txt", self.build_script) + self.assertIn("V8LibcxxMemoryResourcePatch.txt", self.workflow) + self.assertIn("archive_has_memory_resource", self.workflow) + self.assertIn("memory_resource\\.o", self.build_script) + self.assertNotIn( + 'CMAKE_CXX_STANDARD_LIBRARIES=$v8_libcxx_archive;$v8_libcxxabi_archive', + self.build_script, + ) + self.assertIn( + 'CMAKE_CXX_STANDARD_LIBRARIES=$v8_libcxx_archive ' + '$v8_libcxxabi_archive -pthread', + self.build_script, + ) + + def test_arm_mac_can_cross_build_intel_runtime(self) -> None: + self.assertIn("macos_arm64_to_x64=true", self.build_script) + self.assertIn('-DCMAKE_OSX_ARCHITECTURES="$macos_architecture"', self.build_script) + self.assertIn("x86_64-apple-darwin", self.build_script) + self.assertIn("rust-std-$rust_version-x86_64-apple-darwin", self.build_script) + self.assertIn("dotnet_architecture: x64", self.workflow) + self.assertIn("architecture: ${{ matrix.dotnet_architecture }}", self.workflow) + self.assertIn("RUNNER_TOOL_CACHE", self.build_script) + self.assertIn("--retry-all-errors", self.build_script) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_native_runtime_workflow_policy.py b/scripts/tests/test_native_runtime_workflow_policy.py index fd9f83236..2a9bb06c6 100644 --- a/scripts/tests/test_native_runtime_workflow_policy.py +++ b/scripts/tests/test_native_runtime_workflow_policy.py @@ -119,7 +119,7 @@ def test_v8_windows_environment_boundary_is_wired_into_build_and_ci(self) -> Non self.assertGreaterEqual( package_workflow.count( "hashFiles(matrix.v8_cache_script, matrix.v8_cache_patch, " - "'scripts/V8WindowsEnvironment.psm1')" + "'scripts/V8WindowsEnvironment.psm1'," ), 2, ) @@ -139,33 +139,14 @@ def test_v8_windows_environment_boundary_is_wired_into_build_and_ci(self) -> Non self.assertEqual(restore_keys.count("matrix.rid != 'win-x64'"), 2) self.assertNotIn("\n webscene-v8-sdk-", restore_keys) - def test_linux_crash_diagnostics_honor_the_selected_v8_root(self) -> None: + def test_linux_cross_build_uses_the_selected_v8_root(self) -> None: wrapper = ( - ROOT / "scripts/build-native-engine-runtime-linux-container.sh" + ROOT / "scripts/build-linux-native-runtime.sh" ).read_text(encoding="utf-8") - - self.assertIn('--v8-root) v8_root="${arguments[++index]:-}"', wrapper) - self.assertIn( - 'build_variant="-$html_parser-$css_parser-$selector_parser-' - '$dom_bindings-$v8_snapshot"', - wrapper, - ) - self.assertIn('build_variant+=-inspector', wrapper) - self.assertIn( - 'build_dir="$repo_root/artifacts/native-engine-runtime-build/' - 'linux-x64$build_variant"', - wrapper, - ) - self.assertIn( - 'icu_data="$v8_root/out/x64/$v8_configuration/icudtl.dat"', - wrapper, - ) - diagnostic = wrapper.split('native_test_status=0', 1)[1] - self.assertIn('if [[ -d "$build_dir" ]]', diagnostic) - self.assertNotIn( - 'if [[ -f "$icu_data" && -d "$build_dir" ]]', diagnostic - ) - self.assertIn('thread apply all bt', diagnostic) + self.assertIn('v8_root_host="$repo_root/artifacts/native-engine-v8/$rid/v8"', wrapper) + self.assertIn('common_args+=(--v8-root "/workspace/artifacts/native-engine-v8/$rid/v8")', wrapper) + self.assertIn('scripts/build-native-engine-runtime.sh "${common_args[@]}"', wrapper) + self.assertIn('scripts/verify-linux-native-abi.py', wrapper) def test_release_package_gate_covers_every_supported_rid(self) -> None: package_workflow = self.workflows[ @@ -187,7 +168,7 @@ def test_release_package_gate_covers_every_supported_rid(self) -> None: "\n publish:\n", 1 )[0] - for rid in ("osx-arm64", "linux-x64", "win-x64"): + for rid in ("osx-arm64", "osx-x64", "linux-arm64", "linux-x64", "win-x64"): with self.subTest(rid=rid): self.assertIn(f"rid: {rid}", native) self.assertIn(f"--expected-rid {rid}", required) diff --git a/scripts/tests/test_verify_cross_rid_compatibility.py b/scripts/tests/test_verify_cross_rid_compatibility.py index 12c925ed2..3b987d00e 100644 --- a/scripts/tests/test_verify_cross_rid_compatibility.py +++ b/scripts/tests/test_verify_cross_rid_compatibility.py @@ -12,7 +12,7 @@ REPOSITORY_ROOT = pathlib.Path(__file__).resolve().parents[2] VERIFIER = REPOSITORY_ROOT / "scripts" / "verify-cross-rid-compatibility.py" -RIDS = ("osx-arm64", "linux-x64", "win-x64") +RIDS = ("osx-arm64", "osx-x64", "linux-arm64", "linux-x64", "win-x64") class CrossRidCompatibilityVerifierTests(unittest.TestCase): diff --git a/scripts/tests/test_verify_linux_native_abi.py b/scripts/tests/test_verify_linux_native_abi.py new file mode 100644 index 000000000..6f6fa0d64 --- /dev/null +++ b/scripts/tests/test_verify_linux_native_abi.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import importlib.util +import pathlib +import unittest + + +SCRIPT = pathlib.Path(__file__).resolve().parents[1] / "verify-linux-native-abi.py" +SPEC = importlib.util.spec_from_file_location("verify_linux_native_abi", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def elf_text(machine: str = "AArch64", glibc: str = "2.27", *, runpath: bool = False) -> str: + path_line = " 0x0 (RUNPATH) Library runpath: [/workspace/out]" if runpath else "" + return f""" + Machine: {machine} + 0x0 (NEEDED) Shared library: [libc.so.6] + 0x0 (NEEDED) Shared library: [libstdc++.so.6] + {path_line} + Name: GLIBC_{glibc} + Name: GLIBCXX_3.4.24 + Name: CXXABI_1.3.11 + 42: 0 8 FUNC GLOBAL DEFAULT 12 webscene_engine_get_abi_version +""" + + +class LinuxNativeAbiVerifierTests(unittest.TestCase): + def test_accepts_arm64_at_contract_ceiling(self) -> None: + report = MODULE.verify_text(elf_text(), "linux-arm64", "2.27", "3.4.24", "1.3.11") + self.assertEqual("pass", report["status"], report) + + def test_rejects_newer_glibc(self) -> None: + report = MODULE.verify_text(elf_text(glibc="2.28"), "linux-arm64", "2.27", "3.4.24", "1.3.11") + self.assertEqual("fail", report["status"]) + self.assertTrue(any("GLIBC requires 2.28" in issue for issue in report["issues"])) + + def test_rejects_wrong_architecture_and_runpath(self) -> None: + report = MODULE.verify_text(elf_text(machine="Advanced Micro Devices X86-64", runpath=True), "linux-arm64", "2.27", "3.4.24", "1.3.11") + self.assertEqual("fail", report["status"]) + self.assertTrue(any("ELF machine" in issue for issue in report["issues"])) + self.assertTrue(any("RPATH/RUNPATH" in issue for issue in report["issues"])) + + def test_rejects_missing_contract_export(self) -> None: + report = MODULE.verify_text( + elf_text(), "linux-arm64", "2.27", "3.4.24", "1.3.11", + {"webscene_engine_get_abi_version", "webscene_engine_create"}, + ) + self.assertEqual("fail", report["status"]) + self.assertTrue(any("webscene_engine_create" in issue for issue in report["issues"])) + + def test_rejects_interpreter_on_shared_library(self) -> None: + text = elf_text() + "\n[Requesting program interpreter: /lib/ld-linux-aarch64.so.1]\n" + report = MODULE.verify_text(text, "linux-arm64", "2.27", "3.4.24", "1.3.11") + self.assertEqual("fail", report["status"]) + self.assertTrue(any("ELF interpreter" in issue for issue in report["issues"])) + + def test_accepts_glibc_architecture_loader_dependency(self) -> None: + text = elf_text() + "\n 0x0 (NEEDED) Shared library: [ld-linux-aarch64.so.1]\n" + report = MODULE.verify_text(text, "linux-arm64", "2.27", "3.4.24", "1.3.11") + self.assertEqual("pass", report["status"], report) + + def test_rejects_dynamic_openssl_and_zlib_dependencies(self) -> None: + text = elf_text() + """ + 0x0 (NEEDED) Shared library: [libssl.so.1.1] + 0x0 (NEEDED) Shared library: [libcrypto.so.1.1] + 0x0 (NEEDED) Shared library: [libz.so.1] +""" + report = MODULE.verify_text(text, "linux-arm64", "2.27", "3.4.24", "1.3.11") + self.assertEqual("fail", report["status"]) + self.assertTrue(any("libssl.so.1.1" in issue for issue in report["issues"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tests/test_verify_native_payload_reproducibility.py b/scripts/tests/test_verify_native_payload_reproducibility.py new file mode 100644 index 000000000..49e9b0bcc --- /dev/null +++ b/scripts/tests/test_verify_native_payload_reproducibility.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import importlib.util +import pathlib +import tempfile +import unittest +import zipfile + + +SCRIPT = pathlib.Path(__file__).resolve().parents[1] / "verify-native-payload-reproducibility.py" +SPEC = importlib.util.spec_from_file_location("verify_native_payload_reproducibility", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class NativePayloadReproducibilityTests(unittest.TestCase): + def package(self, root: pathlib.Path, name: str, payload: bytes) -> pathlib.Path: + package = root / name + with zipfile.ZipFile(package, "w") as archive: + archive.writestr("runtimes/linux-x64/native/libwebscene_native_engine.so", payload) + archive.writestr("metadata.txt", name) + return package + + def test_ignores_package_container_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = pathlib.Path(temporary) + first = self.package(root, "first.nupkg", b"same") + second = self.package(root, "second.nupkg", b"same") + self.assertEqual(MODULE.payload_hashes(first, "linux-x64"), MODULE.payload_hashes(second, "linux-x64")) + + def test_detects_payload_change(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = pathlib.Path(temporary) + first = self.package(root, "first.nupkg", b"first") + second = self.package(root, "second.nupkg", b"second") + self.assertNotEqual(MODULE.payload_hashes(first, "linux-x64"), MODULE.payload_hashes(second, "linux-x64")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/v8-patch-workspace.sh b/scripts/v8-patch-workspace.sh index 45ec706bb..68cd53cb8 100644 --- a/scripts/v8-patch-workspace.sh +++ b/scripts/v8-patch-workspace.sh @@ -44,9 +44,14 @@ webscene_restore_v8_patches() { "$v8_root/build" "$repo_root/third-party/v8-patches/BuildPatch.txt" webscene_restore_patch_if_applied \ "$v8_root/build" "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8BuildNoCrelPatch.txt" + webscene_restore_patch_if_applied \ + "$v8_root/buildtools" "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8LibcxxMemoryResourcePatch.txt" webscene_restore_patch_if_applied \ "$v8_root/third_party/icu" "$repo_root/third-party/v8-patches/ICUPatch.txt" webscene_restore_patch_if_applied \ "$v8_root/third_party/partition_alloc" \ "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocMacVisibilityPatch.txt" + webscene_restore_patch_if_applied \ + "$v8_root/third_party/partition_alloc/src" \ + "$repo_root/packaging/WebScene.NativeEngine.Runtime/patches/V8PartitionAllocGlibc227Arm64Patch.txt" } diff --git a/scripts/verify-linux-native-abi.py b/scripts/verify-linux-native-abi.py new file mode 100755 index 000000000..c65eae63d --- /dev/null +++ b/scripts/verify-linux-native-abi.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Verify the architecture, dependency, symbol-version, and export contract of a Linux DSO.""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import subprocess + + +ALLOWED_NEEDED = { + "libc.so.6", "libdl.so.2", "libgcc_s.so.1", "libm.so.6", + "libpthread.so.0", "librt.so.1", "libstdc++.so.6", "libutil.so.1", + # glibc's linker scripts can retain the architecture loader as an + # AS_NEEDED dependency. It is part of the glibc ABI on every target distro. + "ld-linux-aarch64.so.1", "ld-linux-x86-64.so.2", +} +EXPECTED_MACHINES = { + "linux-x64": "Advanced Micro Devices X86-64", + "linux-arm64": "AArch64", +} +EXPECTED_INTERPRETERS = { + # Runtime payloads are shared libraries, not PIE executables. A PT_INTERP + # segment would make the payload directly executable and is never valid. + "linux-x64": "", + "linux-arm64": "", +} + + +def version_tuple(value: str) -> tuple[int, ...]: + return tuple(int(part) for part in value.split(".")) + + +def collect_versions(text: str, namespace: str) -> set[str]: + return set(re.findall(rf"\b{re.escape(namespace)}_([0-9]+(?:\.[0-9]+)+)\b", text)) + + +def verify_text( + text: str, + rid: str, + max_glibc: str, + max_glibcxx: str, + max_cxxabi: str, + required_exports: set[str] | None = None, +) -> dict[str, object]: + issues: list[str] = [] + machine_match = re.search(r"^\s*Machine:\s*(.+?)\s*$", text, re.MULTILINE) + machine = machine_match.group(1) if machine_match else "" + if machine != EXPECTED_MACHINES[rid]: + issues.append(f"ELF machine is {machine!r}; expected {EXPECTED_MACHINES[rid]!r}") + + interpreter_match = re.search(r"Requesting program interpreter:\s*([^\]]+)\]", text) + interpreter = interpreter_match.group(1) if interpreter_match else "" + if interpreter != EXPECTED_INTERPRETERS[rid]: + issues.append( + f"ELF interpreter is {interpreter!r}; expected {EXPECTED_INTERPRETERS[rid]!r}" + ) + + needed = set(re.findall(r"\(NEEDED\).*?\[(.+?)\]", text)) + unexpected_needed = sorted(needed - ALLOWED_NEEDED) + if unexpected_needed: + issues.append("unexpected DT_NEEDED libraries: " + ", ".join(unexpected_needed)) + if re.search(r"\((?:RPATH|RUNPATH)\)", text): + issues.append("RPATH/RUNPATH is not permitted") + if "/crossrootfs/" in text or "/workspace/" in text: + issues.append("build or sysroot path leaked into ELF metadata") + + ceilings = {"GLIBC": max_glibc, "GLIBCXX": max_glibcxx, "CXXABI": max_cxxabi} + observed: dict[str, list[str]] = {} + for namespace, ceiling in ceilings.items(): + versions = sorted(collect_versions(text, namespace), key=version_tuple) + observed[namespace] = versions + too_new = [value for value in versions if version_tuple(value) > version_tuple(ceiling)] + if too_new: + issues.append(f"{namespace} requires {too_new[-1]}; maximum is {ceiling}") + + exports = set(re.findall(r"\bGLOBAL\s+DEFAULT\s+\d+\s+(webscene_[A-Za-z0-9_]+)\b", text)) + missing_exports = sorted((required_exports or {"webscene_engine_get_abi_version"}) - exports) + if missing_exports: + issues.append("missing required exports: " + ", ".join(missing_exports)) + + return { + "schemaVersion": 1, + "status": "pass" if not issues else "fail", + "runtimeIdentifier": rid, + "machine": machine, + "interpreter": interpreter, + "needed": sorted(needed), + "symbolVersions": observed, + "limits": ceilings, + "issues": issues, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("library", type=pathlib.Path) + parser.add_argument("--rid", choices=sorted(EXPECTED_MACHINES), required=True) + parser.add_argument("--max-glibc", default="2.27") + parser.add_argument("--max-glibcxx", default="3.4.24") + parser.add_argument("--max-cxxabi", default="1.3.11") + parser.add_argument( + "--exports-file", + type=pathlib.Path, + default=pathlib.Path(__file__).resolve().parents[1] + / "experiments/WebScene.NativeEngine.Probe/native/webscene_native_engine.exports", + ) + parser.add_argument("--output", type=pathlib.Path) + args = parser.parse_args() + if not args.library.is_file(): + parser.error(f"library does not exist: {args.library}") + if not args.exports_file.is_file(): + parser.error(f"exports file does not exist: {args.exports_file}") + required_exports = { + line.strip().removeprefix("_") + for line in args.exports_file.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + completed = subprocess.run( + ["readelf", "--wide", "-h", "-l", "-d", "--version-info", "--dyn-syms", str(args.library)], + check=False, capture_output=True, text=True, + ) + if completed.returncode: + raise RuntimeError(completed.stderr.strip() or "readelf failed") + report = verify_text( + completed.stdout, + args.rid, + args.max_glibc, + args.max_glibcxx, + args.max_cxxabi, + required_exports, + ) + rendered = json.dumps(report, indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-native-payload-reproducibility.py b/scripts/verify-native-payload-reproducibility.py new file mode 100755 index 000000000..2d583c7da --- /dev/null +++ b/scripts/verify-native-payload-reproducibility.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Compare runtime payload bytes from two independently produced NuGet packages.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import pathlib +import zipfile + + +def payload_hashes(package: pathlib.Path, rid: str) -> dict[str, str]: + prefix = f"runtimes/{rid}/native/" + with zipfile.ZipFile(package) as archive: + return { + name.removeprefix(prefix): hashlib.sha256(archive.read(name)).hexdigest() + for name in sorted(archive.namelist()) + if name.startswith(prefix) and not name.endswith("/") + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("first", type=pathlib.Path) + parser.add_argument("second", type=pathlib.Path) + parser.add_argument("--rid", choices=("linux-x64", "linux-arm64"), required=True) + parser.add_argument("--output", type=pathlib.Path) + args = parser.parse_args() + first = payload_hashes(args.first, args.rid) + second = payload_hashes(args.second, args.rid) + report = { + "schemaVersion": 1, + "status": "pass" if first == second else "fail", + "runtimeIdentifier": args.rid, + "first": first, + "second": second, + } + rendered = json.dumps(report, indent=2) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8") + print(rendered, end="") + return 0 if report["status"] == "pass" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify-release-packages.py b/scripts/verify-release-packages.py index 998fdc407..3f747daca 100755 --- a/scripts/verify-release-packages.py +++ b/scripts/verify-release-packages.py @@ -27,13 +27,15 @@ "WebScene.Sdk.Avalonia", "WebScene.Sdk.Uno", } -DEFAULT_NATIVE_RIDS = {"osx-arm64", "linux-x64", "win-x64"} +DEFAULT_NATIVE_RIDS = {"osx-arm64", "osx-x64", "linux-arm64", "linux-x64", "win-x64"} NATIVE_V8_REVISIONS = { "osx-arm64": "15.3.10", + "osx-x64": "15.3.10", + "linux-arm64": "15.3.10", "linux-x64": "15.3.10", "win-x64": "15.3.10", } -PARTITION_ALLOC_NATIVE_RIDS = {"osx-arm64", "linux-x64", "win-x64"} +PARTITION_ALLOC_NATIVE_RIDS = {"osx-arm64", "osx-x64", "linux-arm64", "linux-x64", "win-x64"} REPOSITORY_URL = "https://github.com/wieslawsoltes/WebScene" REQUIRED_PACKAGE_TAGS = {"webscene", "web-ui", "native-ui"} @@ -340,6 +342,18 @@ def validate_native_runtime( "cHeaderFileName": "webscene_native_engine.h", } ) + if runtime_identifier == "linux-x64": + expected.update({ + "builderIdentity": "webscene-linux-glibc-v1", + "targetTriple": "x86_64-linux-gnu", + "glibcBaseline": "2.27", + }) + elif runtime_identifier == "linux-arm64": + expected.update({ + "builderIdentity": "webscene-linux-glibc-v1", + "targetTriple": "aarch64-linux-gnu", + "glibcBaseline": "2.27", + }) for name, value in expected.items(): if manifest.get(name) != value: raise RuntimeError( diff --git a/src/WebScene.Backend.Avalonia/NativeCanvasSceneRenderer.cs b/src/WebScene.Backend.Avalonia/NativeCanvasSceneRenderer.cs index f439aeb06..435286f73 100644 --- a/src/WebScene.Backend.Avalonia/NativeCanvasSceneRenderer.cs +++ b/src/WebScene.Backend.Avalonia/NativeCanvasSceneRenderer.cs @@ -4344,15 +4344,19 @@ private NativeTextShaping.CanvasFontDescription ConfigureFont( if (generic is "-apple-system" or "blinkmacsystemfont" or "system-ui" or "sans-serif") { - family = OperatingSystem.IsMacOS() ? ".AppleSystemUIFont" : "Arial"; + family = OperatingSystem.IsMacOS() + ? ".AppleSystemUIFont" + : OperatingSystem.IsWindows() ? "Arial" : "sans-serif"; } else if (generic == "serif") { - family = "Times New Roman"; + family = OperatingSystem.IsLinux() ? "serif" : "Times New Roman"; } else if (generic == "monospace") { - family = OperatingSystem.IsMacOS() ? "Menlo" : "Consolas"; + family = OperatingSystem.IsMacOS() + ? "Menlo" + : OperatingSystem.IsWindows() ? "Consolas" : "monospace"; } var candidate = SKTypeface.FromFamilyName( family, diff --git a/src/WebScene.Backend.Avalonia/NativeTextShaping.cs b/src/WebScene.Backend.Avalonia/NativeTextShaping.cs index f207909c7..cf5d683c4 100644 --- a/src/WebScene.Backend.Avalonia/NativeTextShaping.cs +++ b/src/WebScene.Backend.Avalonia/NativeTextShaping.cs @@ -476,9 +476,15 @@ internal static SKTypeface ResolveTypeface( ? ".AppleSystemUIFont" : OperatingSystem.IsWindows() ? "Segoe UI" : "sans-serif"; else if (genericFamily == "sans-serif") - family = OperatingSystem.IsMacOS() ? "Helvetica" : "Arial"; - else if (genericFamily == "serif") family = "Times New Roman"; - else if (genericFamily == "monospace") family = OperatingSystem.IsMacOS() ? "Menlo" : "Consolas"; + family = OperatingSystem.IsMacOS() + ? "Helvetica" + : OperatingSystem.IsWindows() ? "Arial" : "sans-serif"; + else if (genericFamily == "serif") + family = OperatingSystem.IsLinux() ? "serif" : "Times New Roman"; + else if (genericFamily == "monospace") + family = OperatingSystem.IsMacOS() + ? "Menlo" + : OperatingSystem.IsWindows() ? "Consolas" : "monospace"; var candidate = SKTypeface.FromFamilyName( family, diff --git a/tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj b/tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj index eb8818854..a50370e89 100644 --- a/tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj +++ b/tests/WebPlatformSubset/runner/WebScene.WebPlatformSubset.Runner.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net8.0;net10.0 enable enable false diff --git a/tests/WebPlatformSubset/webscene-component-profile.json b/tests/WebPlatformSubset/webscene-component-profile.json index d54d8829f..d247b3df2 100644 --- a/tests/WebPlatformSubset/webscene-component-profile.json +++ b/tests/WebPlatformSubset/webscene-component-profile.json @@ -2551,7 +2551,7 @@ "secondColor": "#0000ff", "axis": "horizontal", "minimumPixels": 5, - "maximumPixels": 15, + "maximumPixels": 24, "description": "generated inline whitespace remains visible between differently weighted runs" }, { diff --git a/tests/WebScene.Architecture.Tests/ReleaseCompatibilityGateTests.cs b/tests/WebScene.Architecture.Tests/ReleaseCompatibilityGateTests.cs index df5281908..d06eac79f 100644 --- a/tests/WebScene.Architecture.Tests/ReleaseCompatibilityGateTests.cs +++ b/tests/WebScene.Architecture.Tests/ReleaseCompatibilityGateTests.cs @@ -211,7 +211,7 @@ public void RuntimeWorkflowRunsForProfileChangesAndPublishesPerRidEvidence() workflow, StringComparison.Ordinal); Assert.Contains( - "needs: [metadata, packages, native, required-evidence]", + "needs: [metadata, packages, native, linux-arm64-finalize, required-evidence]", workflow, StringComparison.Ordinal); Assert.Contains( @@ -261,7 +261,7 @@ public void RuntimePublicationRequiresSuccessfulCiForTheExactCommit() Assert.Contains("--status completed", workflow, StringComparison.Ordinal); Assert.Contains("if [[ \"$conclusion\" != success ]]", workflow, StringComparison.Ordinal); Assert.Contains( - "needs: [metadata, consumer, release-ci-gate]", + "needs: [metadata, consumer, linux-floor-smoke, release-ci-gate]", workflow, StringComparison.Ordinal); Assert.Contains("fail-fast: false", ciWorkflow, StringComparison.Ordinal); diff --git a/tests/WebScene.Backend.Avalonia.Tests/NativeTextShapingTests.cs b/tests/WebScene.Backend.Avalonia.Tests/NativeTextShapingTests.cs index 125fe7434..19a9fa79e 100644 --- a/tests/WebScene.Backend.Avalonia.Tests/NativeTextShapingTests.cs +++ b/tests/WebScene.Backend.Avalonia.Tests/NativeTextShapingTests.cs @@ -9,6 +9,20 @@ namespace WebScene.Backend.Avalonia.Tests; [Collection("Native web-font cache")] public sealed class NativeTextShapingTests { + [Fact] + public void LinuxGenericFamiliesResolveThroughFontconfig() + { + if (!OperatingSystem.IsLinux()) return; + + var sansSerif = NativeTextShaping.ResolveTypeface("sans-serif", 400); + var serif = NativeTextShaping.ResolveTypeface("serif", 400); + var monospace = NativeTextShaping.ResolveTypeface("monospace", 400); + + Assert.NotEqual(sansSerif.FamilyName, serif.FamilyName); + Assert.NotEqual(sansSerif.FamilyName, monospace.FamilyName); + Assert.NotEqual(serif.FamilyName, monospace.FamilyName); + } + [Fact] public void WindowsGenericFamiliesKeepSystemUiAndSansSerifDistinct() { diff --git a/tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj b/tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj index bfd02ad22..2adec778b 100644 --- a/tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj +++ b/tests/WebScene.Runtime.PackageSmoke/WebScene.Runtime.PackageSmoke.csproj @@ -1,7 +1,7 @@ Exe - net8.0 + net8.0;net10.0 enable enable false