diff --git a/.github/actions/onnx-build-katago/action.yml b/.github/actions/onnx-build-katago/action.yml new file mode 100644 index 0000000000..f8eefdd318 --- /dev/null +++ b/.github/actions/onnx-build-katago/action.yml @@ -0,0 +1,107 @@ +name: Build and test KataGo (ONNX backend) +description: > + Configure, build, run `katago runtests`, verify the ORT backend is wired up, and stage a + self-contained runnable directory under release/ (katago binary + ORT runtime + EP + runtimes + example config). Expects onnx-prepare-ort to have populated + deps/install/{ort,protobuf,zlib} and the MSVC environment to be ready on Windows. +inputs: + ort_root: + description: Path to the ORT install tree (deps/install/ort) + required: true + ep: + description: Execution provider, used for EP-specific runtime staging + required: true + +runs: + using: composite + steps: + - name: Configure KataGo (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + cmake -S cpp -B cpp\build -G Ninja -DCMAKE_BUILD_TYPE=Release -DUSE_BACKEND=ONNX ` + -DONNXRUNTIME_ROOT=${{ inputs.ort_root }} ` + -DProtobuf_PROTOC_EXECUTABLE=${{ github.workspace }}\deps\install\protobuf\bin\protoc.exe ` + -DProtobuf_INCLUDE_DIR=${{ github.workspace }}\deps\install\protobuf\include ` + -DProtobuf_LIBRARY=${{ github.workspace }}\deps\install\protobuf\lib\libprotobuf.lib ` + -DZLIB_INCLUDE_DIR=${{ github.workspace }}\deps\install\zlib\include ` + -DZLIB_LIBRARY=${{ github.workspace }}\deps\install\zlib\lib\zlibstatic.lib + + - name: Configure KataGo (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + # Use $GITHUB_WORKSPACE (not the github.workspace context): it resolves correctly + # both on plain runners (/home/runner/...) and inside a container job (/__w/...). + cmake -S cpp -B cpp/build -G Ninja -DCMAKE_BUILD_TYPE=Release -DUSE_BACKEND=ONNX \ + -DONNXRUNTIME_ROOT="$GITHUB_WORKSPACE/deps/install/ort" \ + -DProtobuf_PROTOC_EXECUTABLE="$GITHUB_WORKSPACE/deps/install/protobuf/bin/protoc" \ + -DProtobuf_INCLUDE_DIR="$GITHUB_WORKSPACE/deps/install/protobuf/include" \ + -DProtobuf_LIBRARY="$GITHUB_WORKSPACE/deps/install/protobuf/lib/libprotobuf.a" \ + -DZLIB_INCLUDE_DIR="$GITHUB_WORKSPACE/deps/install/zlib/include" \ + -DZLIB_LIBRARY="$GITHUB_WORKSPACE/deps/install/zlib/lib/libz.a" + + - name: Build KataGo + shell: bash + run: cmake --build cpp/build + + - name: Run tests + shell: bash + run: | + if [ "${{ runner.os }}" = "Windows" ]; then ./cpp/build/katago.exe runtests; else ./cpp/build/katago runtests; fi + + - name: Verify backend wiring + shell: bash + run: | + # Command substitution flattens multi-line output: testing the raw array for a + # substring is unreliable, so check the flattened string with grep. + if [ "${{ runner.os }}" = "Windows" ]; then OUT=$(cpp/build/katago.exe version 2>&1); else OUT=$(cpp/build/katago version 2>&1); fi + if ! echo "$OUT" | grep -q "ONNX Runtime"; then + echo "unexpected backend version output: $OUT" + exit 1 + fi + echo "$OUT" | head -8 + + - name: Stage release directory (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "release" | Out-Null + Copy-Item "cpp\build\katago.exe" "release\" + Get-ChildItem "deps\install\ort\bin\*.dll" -ErrorAction SilentlyContinue | Copy-Item -Destination "release\" -Force + # EP-specific runtime DLLs + if ("${{ inputs.ep }}" -eq "openvino") { + Get-ChildItem "deps\install\ort\lib\onnxruntime_providers_openvino.dll" -ErrorAction SilentlyContinue | Copy-Item -Destination "release\" -Force + $ovBin = "deps\openvino\runtime\bin\intel64\Release" + if (Test-Path $ovBin) { + Copy-Item "$ovBin\*.dll" "release\" + Copy-Item "$ovBin\*.json" "release\" + } else { + Write-Warning "OpenVINO runtime bin dir not found at $ovBin" + } + $tbb = Get-ChildItem "deps\openvino" -Recurse -Filter "tbb12.dll" -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($tbb) { Copy-Item $tbb.FullName "release\" } + } + Copy-Item "cpp\configs\gtp_example.cfg" "release\" + Write-Output "--- release contents ---" + Get-ChildItem "release" | Select-Object Name, Length + + - name: Stage release directory (Linux) + if: runner.os == 'Linux' + shell: bash + run: | + mkdir -p release + cp cpp/build/katago release/ + cp deps/install/ort/lib/libonnxruntime*.so* release/ 2>/dev/null || true + # EP-specific runtime libs (TensorRT/CUDA/ROCm, etc.) go here once wired up. + cp cpp/configs/gtp_example.cfg release/ + # katago's build-time DT_RUNPATH points at the CI workspace + # (deps/install/ort/lib), which no longer exists once the artifact is downloaded + # elsewhere. Rewrite it to $ORIGIN so the binary finds the sibling + # libonnxruntime.so* in the release dir, matching the Windows DLL convention. + if ! command -v patchelf >/dev/null 2>&1; then + # Ubuntu runners have sudo; the NGC TensorRT container runs as root without it. + if command -v sudo >/dev/null 2>&1; then sudo apt-get install -y patchelf >/dev/null; else apt-get install -y patchelf >/dev/null; fi + fi + patchelf --set-rpath '$ORIGIN' release/katago + ls -la release diff --git a/.github/actions/onnx-prepare-ort/action.yml b/.github/actions/onnx-prepare-ort/action.yml new file mode 100644 index 0000000000..993138ac2a --- /dev/null +++ b/.github/actions/onnx-prepare-ort/action.yml @@ -0,0 +1,264 @@ +name: Prepare ONNX Runtime +description: > + Fetch or build an ONNX Runtime install tree carrying the requested execution provider, + plus KataGo's own zlib/protobuf deps. Everything lands under deps/install/ and is + cached, so only the first run pays for ORT acquisition (from-source ORT builds are the + expensive case, 1-3h; prebuilt/nuget are minutes). +inputs: + ep: + description: Execution provider to wire up (cpu, directml, openvino, ...) + required: true + mode: + description: > + How to obtain ORT. 'prebuilt' = official release zip (CPU EP ships inside it), + 'nuget' = Microsoft.ML.OnnxRuntime.DirectML package, 'from-source' = build ORT + ourselves with the EP. + required: true + ort_version: + description: ORT release version for prebuilt/nuget modes + required: false + default: "1.28.0" + ort_ref: + description: ORT git ref (tag/SHA) for from-source mode + required: false + default: "" + ov_version: + description: OpenVINO toolkit version (from-source/openvino only) + required: false + default: "" + ov_url: + description: OpenVINO toolkit download URL (from-source/openvino only) + required: false + default: "" + pb_version: + description: protobuf version built for KataGo (3.x, no abseil dependency) + required: false + default: "3.21.12" + pb_tag: + description: > + protobuf release tag for pb_version. protobuf 3.x tags drop the major (3.21.12 + -> v21.12), so this cannot be derived from pb_version. + required: false + default: "21.12" + zlib_version: + description: zlib version built for KataGo + required: false + default: "1.3.1" + +outputs: + ort_root: + description: Path to the prepared ORT install tree + value: ${{ github.workspace }}/deps/install/ort + +runs: + using: composite + steps: + # --------------------------------------------------------------------------- + # Common deps: zlib + protobuf, always built from source and cached separately + # from the ORT tree so a from-source ORT cache miss does not rebuild them. + # --------------------------------------------------------------------------- + - name: Restore common-dep cache + id: cache-common + uses: actions/cache@v4 + with: + path: | + deps/install/zlib + deps/install/protobuf + key: onnx-common-${{ runner.os }}-pb-${{ inputs.pb_version }}-zl-${{ inputs.zlib_version }} + + - name: Build zlib (static, Windows) + if: steps.cache-common.outputs.cache-hit != 'true' && runner.os == 'Windows' + shell: pwsh + run: | + curl.exe -L -o "$env:RUNNER_TEMP\zlib.tar.gz" "https://github.com/madler/zlib/releases/download/v${{ inputs.zlib_version }}/zlib-${{ inputs.zlib_version }}.tar.gz" + tar -xzf "$env:RUNNER_TEMP\zlib.tar.gz" -C "$env:RUNNER_TEMP" + cmake -S "$env:RUNNER_TEMP\zlib-${{ inputs.zlib_version }}" -B "$env:RUNNER_TEMP\zlib-build" -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\deps\install\zlib + cmake --build "$env:RUNNER_TEMP\zlib-build" + cmake --install "$env:RUNNER_TEMP\zlib-build" + + - name: Build zlib (static, Linux) + if: steps.cache-common.outputs.cache-hit != 'true' && runner.os == 'Linux' + shell: bash + run: | + curl -L -o "$RUNNER_TEMP/zlib.tar.gz" "https://github.com/madler/zlib/releases/download/v${{ inputs.zlib_version }}/zlib-${{ inputs.zlib_version }}.tar.gz" + tar -xzf "$RUNNER_TEMP/zlib.tar.gz" -C "$RUNNER_TEMP" + cmake -S "$RUNNER_TEMP/zlib-${{ inputs.zlib_version }}" -B "$RUNNER_TEMP/zlib-build" -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE/deps/install/zlib" + cmake --build "$RUNNER_TEMP/zlib-build" + cmake --install "$RUNNER_TEMP/zlib-build" + + - name: Build protobuf (static, /MD, Windows) + if: steps.cache-common.outputs.cache-hit != 'true' && runner.os == 'Windows' + shell: pwsh + run: | + curl.exe -L -o "$env:RUNNER_TEMP\pb.tar.gz" "https://github.com/protocolbuffers/protobuf/releases/download/v${{ inputs.pb_tag }}/protobuf-cpp-${{ inputs.pb_version }}.tar.gz" + tar -xzf "$env:RUNNER_TEMP\pb.tar.gz" -C "$env:RUNNER_TEMP" + # protobuf_MSVC_STATIC_RUNTIME=OFF is REQUIRED: the protobuf default (/MT) would + # clash with KataGo's /MD Release build (LNK2038) when linking libprotobuf statically. + cmake -S "$env:RUNNER_TEMP\protobuf-${{ inputs.pb_version }}" -B "$env:RUNNER_TEMP\pb-build" -G Ninja ` + -DCMAKE_BUILD_TYPE=Release ` + -Dprotobuf_BUILD_TESTS=OFF ` + -Dprotobuf_BUILD_SHARED_LIBS=OFF ` + -Dprotobuf_MSVC_STATIC_RUNTIME=OFF ` + -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\deps\install\protobuf + cmake --build "$env:RUNNER_TEMP\pb-build" + cmake --install "$env:RUNNER_TEMP\pb-build" + + - name: Build protobuf (static, Linux) + if: steps.cache-common.outputs.cache-hit != 'true' && runner.os == 'Linux' + shell: bash + run: | + curl -L -o "$RUNNER_TEMP/pb.tar.gz" "https://github.com/protocolbuffers/protobuf/releases/download/v${{ inputs.pb_tag }}/protobuf-cpp-${{ inputs.pb_version }}.tar.gz" + tar -xzf "$RUNNER_TEMP/pb.tar.gz" -C "$RUNNER_TEMP" + cmake -S "$RUNNER_TEMP/protobuf-${{ inputs.pb_version }}" -B "$RUNNER_TEMP/pb-build" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_BUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_PREFIX="$GITHUB_WORKSPACE/deps/install/protobuf" + cmake --build "$RUNNER_TEMP/pb-build" + cmake --install "$RUNNER_TEMP/pb-build" + + # --------------------------------------------------------------------------- + # ONNX Runtime install tree, cached per (os, ep, version/ref). + # --------------------------------------------------------------------------- + - name: Restore ORT cache + id: cache-ort + uses: actions/cache@v4 + with: + path: deps/install/ort + key: onnx-ort-${{ runner.os }}-${{ inputs.ep }}-${{ inputs.ort_version }}-${{ inputs.ort_ref }} + + # --- prebuilt: official ORT release package (CPU EP ships inside it): zip on Windows, tgz on Linux --- + - name: Fetch prebuilt ORT (Windows) + if: inputs.mode == 'prebuilt' && runner.os == 'Windows' && steps.cache-ort.outputs.cache-hit != 'true' + shell: pwsh + run: | + New-Item -ItemType Directory -Force -Path "deps\install\ort" | Out-Null + curl.exe -sfL -o "$env:RUNNER_TEMP\ort.zip" "https://github.com/microsoft/onnxruntime/releases/download/v${{ inputs.ort_version }}/onnxruntime-win-x64-${{ inputs.ort_version }}.zip" + Expand-Archive -Path "$env:RUNNER_TEMP\ort.zip" -DestinationPath "$env:RUNNER_TEMP\ort" -Force + $inner = Get-ChildItem "$env:RUNNER_TEMP\ort" -Directory | Select-Object -First 1 + if ($inner) { + Get-ChildItem $inner.FullName | Move-Item -Destination "deps\install\ort" -Force + Remove-Item $inner.FullName -Recurse -Force + } + Remove-Item "$env:RUNNER_TEMP\ort.zip" -Force + # Hoist lib/*.dll into bin/ too: the release-staging step globs bin/. + New-Item -ItemType Directory -Force -Path "deps\install\ort\bin" | Out-Null + Get-ChildItem "deps\install\ort\lib\*.dll" -ErrorAction SilentlyContinue | Copy-Item -Destination "deps\install\ort\bin" -Force + + - name: Fetch prebuilt ORT (Linux) + if: inputs.mode == 'prebuilt' && runner.os == 'Linux' && steps.cache-ort.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p deps/install/ort + curl -sfL -o "$RUNNER_TEMP/ort.tgz" "https://github.com/microsoft/onnxruntime/releases/download/v${{ inputs.ort_version }}/onnxruntime-linux-x64-${{ inputs.ort_version }}.tgz" + tar -xzf "$RUNNER_TEMP/ort.tgz" -C deps/install/ort --strip-components=1 + # No lib/*.dll hoist is needed here: Linux resolves libonnxruntime.so through the + # binary's DT_RUNPATH, which onnx-build-katago rewrites to $ORIGIN at staging time. + ls deps/install/ort + + # --- nuget: Microsoft.ML.OnnxRuntime.DirectML package (DirectML EP) --- + - name: Fetch DirectML NuGet package (Windows) + if: inputs.mode == 'nuget' && runner.os == 'Windows' && steps.cache-ort.outputs.cache-hit != 'true' + shell: pwsh + run: | + $src = "$env:RUNNER_TEMP\dml" + New-Item -ItemType Directory -Force -Path "$src" | Out-Null + curl.exe -sfL -o "$src\dml.nupkg" "https://api.nuget.org/v3-flatcontainer/microsoft.ml.onnxruntime.directml/${{ inputs.ort_version }}/microsoft.ml.onnxruntime.directml.${{ inputs.ort_version }}.nupkg" + tar -xzf "$src\dml.nupkg" -C "$src" + $root = "deps\install\ort" + New-Item -ItemType Directory -Force -Path "$root\include","$root\lib","$root\bin" | Out-Null + Copy-Item "$src\build\native\include\*" "$root\include" -Force + Copy-Item "$src\runtimes\win-x64\native\onnxruntime.lib" "$root\lib" -Force + Get-ChildItem "$src\runtimes\win-x64\native\*.dll" | Copy-Item -Destination "$root\lib" -Force + Get-ChildItem "$root\lib\*.dll" | Copy-Item -Destination "$root\bin" -Force + + # --- from-source: build ORT ourselves with the requested EP --- + - name: Checkout ONNX Runtime source + if: inputs.mode == 'from-source' && steps.cache-ort.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: microsoft/onnxruntime + ref: ${{ inputs.ort_ref }} + path: deps/onnxruntime + submodules: recursive + + - name: Fetch OpenVINO toolkit (Windows) + if: inputs.mode == 'from-source' && inputs.ep == 'openvino' && runner.os == 'Windows' && steps.cache-ort.outputs.cache-hit != 'true' + shell: pwsh + run: | + $zip = "$env:RUNNER_TEMP\openvino.zip" + $dest = "deps\openvino" + Invoke-WebRequest -Uri "${{ inputs.ov_url }}" -OutFile $zip + Expand-Archive -Path $zip -DestinationPath $dest + # The zip contains a single top-level directory of the same name; hoist its contents up. + $inner = Get-ChildItem -Path $dest -Directory | Select-Object -First 1 + if ($inner) { + Get-ChildItem -Path $inner.FullName | Move-Item -Destination $dest -Force + Remove-Item -Path $inner.FullName -Recurse -Force + } + Remove-Item -Path $zip -Force + $sv = Get-ChildItem "deps\openvino" -Recurse -Filter "setupvars.bat" | Select-Object -First 1 + if (-not $sv) { throw "setupvars.bat not found under deps\openvino" } + echo "OV_SETUPVARS=$($sv.FullName)" >> $env:GITHUB_ENV + + - name: Build ONNX Runtime with OpenVINO EP + if: inputs.mode == 'from-source' && inputs.ep == 'openvino' && steps.cache-ort.outputs.cache-hit != 'true' + shell: cmd + working-directory: deps/onnxruntime + run: | + call "%OV_SETUPVARS%" + python tools\ci_build\build.py --build_dir build --config Release --use_openvino GPU --build_shared_lib --skip_tests --parallel --compile_no_warning_as_error --cmake_generator Ninja --cmake_extra_defines CMAKE_INSTALL_PREFIX=%CD%\..\install\ort + if errorlevel 1 exit /b 1 + cmake --install build\Release --config Release + if errorlevel 1 exit /b 1 + + - name: Build ONNX Runtime with TensorRT EP (Linux container) + if: inputs.mode == 'from-source' && inputs.ep == 'tensorrt' && runner.os == 'Linux' && steps.cache-ort.outputs.cache-hit != 'true' + shell: bash + working-directory: deps/onnxruntime + run: | + # Runs inside the official NGC TensorRT container + # (nvcr.io/nvidia/tensorrt:25.03-py3 = CUDA 12.8 + TensorRT 10.9, the combo ORT is + # tested against), which preinstalls CUDA/cuDNN/TensorRT under /usr/local/cuda and + # /usr/lib/x86_64-linux-gnu. The container ships cmake + python3; the workflow's + # bootstrap step adds ninja. github-hosted runners have no GPU, so this verifies + # build + EP wiring only - real GPU inference is validated on a GPU machine. + # Build only for sm_89 (RTX 4090): ORT's default 10-arch matrix is huge and slow + # on a no-GPU CI runner. Tune per the GPU you validate on. + python3 tools/ci_build/build.py --build_dir build --config Release \ + --use_tensorrt --use_cuda \ + --cuda_home /usr/local/cuda \ + --cudnn_home /usr/lib/x86_64-linux-gnu \ + --tensorrt_home /usr/lib/x86_64-linux-gnu \ + --build_shared_lib --skip_tests --parallel \ + --cmake_generator Ninja --allow_running_as_root --skip_submodule_sync \ + --cmake_extra_defines "CMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/deps/install/ort" "CMAKE_CUDA_ARCHITECTURES=89" + cmake --install build/Release --config Release + + # NOTE: MIGraphX from-source build slots in here once validated (needs ROCm; deferred). + + # Save caches explicitly: actions/cache@v4's post-save does not reliably run inside + # composite actions, so without these every run would rebuild zlib/protobuf and ORT. + # Only save when this run actually built the dep (restore cache-missed); the key must + # match the restore key above. + - name: Save common-dep cache + if: steps.cache-common.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: | + deps/install/zlib + deps/install/protobuf + key: onnx-common-${{ runner.os }}-pb-${{ inputs.pb_version }}-zl-${{ inputs.zlib_version }} + + - name: Save ORT cache + if: steps.cache-ort.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: deps/install/ort + key: onnx-ort-${{ runner.os }}-${{ inputs.ep }}-${{ inputs.ort_version }}-${{ inputs.ort_ref }} + + - name: Report ORT install tree + shell: bash + run: | + ls -la deps/install/ort + ls deps/install/ort/bin 2>/dev/null || ls deps/install/ort/lib 2>/dev/null || true diff --git a/.github/workflows/onnx-backend.yml b/.github/workflows/onnx-backend.yml new file mode 100644 index 0000000000..197dfe6326 --- /dev/null +++ b/.github/workflows/onnx-backend.yml @@ -0,0 +1,203 @@ +# ONNX backend CI. Builds KataGo (USE_BACKEND=ONNX) against an ONNX Runtime carrying the +# requested execution provider, runs `katago runtests`, and uploads a self-contained +# runnable directory as an artifact for easy download. +# +# Execution providers differ only in how ONNX Runtime is obtained; that difference lives +# in .github/actions/onnx-prepare-ort/action.yml (one row in the matrix below per EP): +# - prebuilt : official ORT release package (zip on Windows, tgz on Linux; the CPU +# EP ships inside it) +# - nuget : Microsoft.ML.OnnxRuntime.DirectML package (DirectML EP) +# - from-source : ORT built from source with the EP (OpenVINO on Windows, TensorRT in an +# NGC container) +# +# Trigger policy by tier: +# - fast EPs (prebuilt/nuget, a few minutes) run on PR + master push + manual dispatch, +# so the ONNX backend keeps a cheap always-on regression guard. +# - slow EPs (from-source ORT builds, 1-3h) run only on manual dispatch, so they never +# burn upstream CI minutes on every PR/commit. +# GitHub-hosted runners have no GPU, so from-source jobs only verify build + EP wiring; +# real GPU inference must be validated on a GPU machine. +# +# Adding a backend = add one matrix row + teach onnx-prepare-ort to fetch/build its ORT +# (and extend the release-staging step in onnx-build-katago if it ships extra runtimes). + +name: ONNX backend build & test + +on: + pull_request: + branches: [ master ] + paths: + - 'cpp/**' + - '.github/workflows/onnx-backend.yml' + - '.github/actions/**' + push: + # ci/onnx-windows is a TEMPORARY trigger so this workflow can be exercised before it + # exists on the default branch (workflow_dispatch requires the file on the default + # branch). Remove it once the branch is validated / merged to master. + branches: [ master, ci/onnx-windows ] + paths: + - 'cpp/**' + - '.github/workflows/onnx-backend.yml' + - '.github/actions/**' + workflow_dispatch: + +concurrency: + group: onnx-backend-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + # from-source ORT pin. 1.29.0 has no release tag (VERSION_NUMBER is 1.29.0 but no + # v1.29.0 tag exists), so pin to the exact commit the ONNX backend was verified against + # locally instead of master, which would drift. Update to re-validate a newer snapshot. + ORT_REF: 7e76a52398ebf966bcbe4a10e552f438059edfce + OV_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/2026.2.1/windows/openvino_toolkit_windows_2026.2.1.21919.ede283a88e3_x86_64.zip + OV_VERSION: 2026.2.1 + +jobs: + build-fast: + name: ${{ matrix.ep }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - { ep: cpu, os: windows-latest, mode: prebuilt, ort_version: "1.28.0" } + - { ep: cpu, os: ubuntu-latest, mode: prebuilt, ort_version: "1.28.0" } + - { ep: directml, os: windows-latest, mode: nuget, ort_version: "1.24.4" } + steps: + - uses: actions/checkout@v4 + + - name: Setup MSVC environment + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Install Ninja (Windows) + if: runner.os == 'Windows' + run: choco install ninja -y --no-progress + + - name: Install Ninja (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y ninja-build + + - name: Prepare ONNX Runtime (${{ matrix.ep }}) + uses: ./.github/actions/onnx-prepare-ort + with: + ep: ${{ matrix.ep }} + mode: ${{ matrix.mode }} + ort_version: ${{ matrix.ort_version }} + + - name: Build & test KataGo + uses: ./.github/actions/onnx-build-katago + with: + ort_root: ${{ github.workspace }}/deps/install/ort + ep: ${{ matrix.ep }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: katago-${{ runner.os }}-onnx-${{ matrix.ep }} + path: release/ + + build-slow: + # from-source ORT builds (1-3h) only on manual dispatch. + if: github.event_name == 'workflow_dispatch' + name: ${{ matrix.ep }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - { ep: openvino, os: windows-latest, mode: from-source } + # - { ep: migraphx, os: ubuntu-latest, mode: from-source } # needs ROCm; deferred (no ROCm on hosted runners) + steps: + - uses: actions/checkout@v4 + + - name: Setup MSVC environment + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Install Ninja (Windows) + if: runner.os == 'Windows' + run: choco install ninja -y --no-progress + + - name: Install Ninja (Linux) + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y ninja-build + + - name: Prepare ONNX Runtime (${{ matrix.ep }}) + uses: ./.github/actions/onnx-prepare-ort + with: + ep: ${{ matrix.ep }} + mode: ${{ matrix.mode }} + ort_ref: ${{ env.ORT_REF }} + ov_version: ${{ env.OV_VERSION }} + ov_url: ${{ env.OV_URL }} + + - name: Build & test KataGo + uses: ./.github/actions/onnx-build-katago + with: + ort_root: ${{ github.workspace }}/deps/install/ort + ep: ${{ matrix.ep }} + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: katago-${{ runner.os }}-onnx-${{ matrix.ep }} + path: release/ + + build-tensorrt: + # ORT from-source TensorRT build (1-2h), dispatch-only. Runs inside the official NGC + # TensorRT container (nvcr.io/nvidia/tensorrt:25.03-py3 = CUDA 12.8 + TensorRT 10.9, the + # combo ORT is built/tested against), so the CUDA/cuDNN/TensorRT SDKs are preinstalled + # and no SDK install step is needed. github-hosted runners have no GPU, so this job + # verifies build + EP wiring only; real GPU inference is validated on a GPU machine. + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + container: + image: nvcr.io/nvidia/tensorrt:25.03-py3 + options: --user root + steps: + # The NGC image ships neither git nor node, and later steps are GitHub JS actions + # (cache, checkout, upload-artifact) that need node. Bootstrap both plus ninja first, + # then clone the repo by hand (checkout@v4 would run before node existed). + - name: Bootstrap container (git, node20, ninja) and checkout + run: | + apt-get update -qq + apt-get install -y -qq git ninja-build curl >/dev/null + curl -fsSL https://deb.nodesource.com/setup_20.x | bash - + apt-get install -y -qq nodejs >/dev/null + node --version + # ORT 1.28 needs CMake >= 3.28; the NGC image ships 3.27. pip cmake lands in + # /usr/local/bin ahead of the bundled one. + pip3 install cmake >/dev/null + cmake --version | head -1 + # The workspace is a docker mount owned by a different uid, so git refuses it + # ("dubious ownership") when KataGo regenerates gitinfo.h during the build. + git config --global --add safe.directory '*' + git clone --depth 1 --branch "${GITHUB_REF_NAME}" "https://github.com/${GITHUB_REPOSITORY}.git" . + + - name: Prepare ONNX Runtime (tensorrt) + uses: ./.github/actions/onnx-prepare-ort + with: + ep: tensorrt + mode: from-source + ort_ref: ${{ env.ORT_REF }} + + - name: Build & test KataGo + uses: ./.github/actions/onnx-build-katago + with: + ort_root: ${{ github.workspace }}/deps/install/ort + ep: tensorrt + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: katago-Linux-onnx-tensorrt + path: release/ diff --git a/Compiling.md b/Compiling.md index abe7de36fc..994494f19f 100644 --- a/Compiling.md +++ b/Compiling.md @@ -152,3 +152,55 @@ As also mentioned in the instructions below but repeated here for visibility, if * Pre-trained neural nets are available at [the main training website](https://katagotraining.org/). * You will probably want to edit `configs/gtp_example.cfg` (see "Tuning for Performance" above). * If using OpenCL, you will want to verify that KataGo is picking up the correct device when you run it (e.g. some systems may have both an Intel CPU OpenCL and GPU OpenCL, if KataGo appears to pick the wrong one, you can correct this by specifying `openclGpuToUse` in `configs/gtp_example.cfg`). + +## ONNX Runtime backend (optional) +The `ONNX` backend runs inference through [ONNX Runtime](https://onnxruntime.ai/), which selects an execution provider at runtime (see the support matrix below). It reuses KataGo's built-in `OnnxModelBuilder` (the same graph emitter the TensorRT backend uses), so its IO protocol and post-processing are identical to TensorRT; only the runtime differs. It is useful when you want to run KataGo on a non-NVIDIA accelerator that already has an ONNX Runtime execution provider, or for cross-vendor benchmarking. + +### Execution provider support matrix + +`onnxProvider` selects an execution provider at runtime. Upstream verification is +**limited to OpenVINO on Windows + Intel GPU**; the other entries below are code +paths that should work but are not continuously tested. + +| Provider | Status | Platform | ORT build flag | Runtime deps | +|---|---|---|---|---| +| `openvino` | Verified (Windows, Intel GPU; see benchmark notes) | Windows / Linux | `--use_openvino GPU` | OpenVINO runtime DLLs, TBB | +| `cpu` | Experimental | All | none (stock ORT) | - | +| `cuda` | Experimental | Windows / Linux | `--use_cuda` | CUDA runtime | +| `tensorrt` | Experimental | Windows / Linux | `--use_tensorrt` | TensorRT | +| `migraphx` | Experimental | Linux (AMD) | `--use_migraphx` | MIGraphX | +| `coreml` | Needs work (build blocked) | macOS | `--use_coreml` | CoreML | + +Status legend: +- **Verified** - covered by CI and/or end-to-end manual testing; numbers, precision + and runtime-dependency deployment are confirmed. +- **Experimental** - the code path exists and the architecture is EP-agnostic, but it + is NOT tested upstream. You must build ONNX Runtime yourself with the matching EP, + and you should validate numerics yourself (e.g. against the `cpu` provider). +- **Needs work** - currently cannot build; see PR notes. + +> **Note**: This backend is more involved to set up than the built-in backends above, because the official prebuilt ONNX Runtime packages do **not** ship the execution providers you may need (e.g. the OpenVINO EP). You generally have to build ONNX Runtime from source with the provider(s) you want enabled. + +### Requirements + * Everything KataGo normally needs (CMake, a C++17 compiler, zlib). + * ONNX Runtime, built from source with the execution provider(s) you intend to use. For the OpenVINO EP, build ONNX Runtime with `--use_openvino` against an installed OpenVINO toolkit. See https://onnxruntime.ai/docs/install/ for build instructions. + * Protobuf. The ONNX graph is serialized as an ONNX `ModelProto`, so `find_package(Protobuf)` must succeed. A protobuf 3.x (no abseil dependency) works; the version bundled in the ONNX Runtime source build tree is known to work. + * If using the OpenVINO EP, the OpenVINO runtime toolkit itself, plus its runtime DLLs at runtime (see below). + +### Compile + * Point CMake at your ONNX Runtime install tree and protobuf, and select the backend: + ``` + cmake -S KataGo/cpp -B KataGo/cpp/build -DUSE_BACKEND=ONNX ^ + -DONNXRUNTIME_ROOT= ^ + -DProtobuf_PROTOC_EXECUTABLE= ^ + -DProtobuf_INCLUDE_DIR= ^ + -DProtobuf_LIBRARY= + cmake --build KataGo/cpp/build -j + ``` + * `-DONNXRUNTIME_ROOT` should contain `include/onnxruntime/`, `lib/onnxruntime.lib` (or `.so`/`.dylib`), and the provider DLLs. + * As with other backends, `-DNO_GIT_REVISION=1` avoids embedding the git hash, and `-DBUILD_DISTRIBUTED=1` enables distributed-training support. + +### Runtime + * The `onnxruntime` shared library must be on your path or beside the executable. + * When using the OpenVINO EP, also deploy the OpenVINO runtime DLLs beside the executable (`openvino.dll`, `openvino_intel_gpu_plugin.dll`, `tbb12.dll`, `cache.json`, etc.), or put them on the system path. + * Configure the provider in `configs/gtp_example.cfg` via the `onnx*` keys, e.g. `onnxProvider=openvino` and `onnxOpenVINODeviceType=GPU`. See the ONNX settings block in `configs/gtp_example.cfg` for the full list. diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index fb8bb130fa..a8c9258757 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -44,7 +44,7 @@ endif() set(BUILD_DISTRIBUTED 0 CACHE BOOL "Build with http support for contributing to distributed training") set(USE_BACKEND CACHE STRING "Neural net backend") string(TOUPPER "${USE_BACKEND}" USE_BACKEND) -set_property(CACHE USE_BACKEND PROPERTY STRINGS "" CUDA TENSORRT OPENCL EIGEN METAL) +set_property(CACHE USE_BACKEND PROPERTY STRINGS "" CUDA TENSORRT OPENCL EIGEN METAL ONNX) set(USE_TCMALLOC 0 CACHE BOOL "Use TCMalloc") set(NO_GIT_REVISION 0 CACHE BOOL "Disable embedding the git revision into the compiled exe") @@ -164,8 +164,13 @@ elseif(USE_BACKEND STREQUAL "EIGEN") set(NEURALNET_BACKEND_SOURCES neuralnet/eigenbackend.cpp ) +elseif(USE_BACKEND STREQUAL "ONNX") + message(STATUS "-DUSE_BACKEND=ONNX, using ONNX Runtime backend.") + set(NEURALNET_BACKEND_SOURCES + neuralnet/onnxbackend.cpp + ) elseif(USE_BACKEND STREQUAL "") - message(WARNING "${ColorBoldRed}WARNING: Using dummy neural net backend, intended for non-neural-net testing only, will fail on any code path requiring a neural net. To use neural net, specify -DUSE_BACKEND=CUDA or -DUSE_BACKEND=TENSORRT or -DUSE_BACKEND=OPENCL or -DUSE_BACKEND=EIGEN to compile with the respective backend.${ColorReset}") + message(WARNING "${ColorBoldRed}WARNING: Using dummy neural net backend, intended for non-neural-net testing only, will fail on any code path requiring a neural net. To use neural net, specify -DUSE_BACKEND=CUDA or -DUSE_BACKEND=TENSORRT or -DUSE_BACKEND=OPENCL or -DUSE_BACKEND=EIGEN or -DUSE_BACKEND=ONNX to compile with the respective backend.${ColorReset}") set(NEURALNET_BACKEND_SOURCES neuralnet/dummybackend.cpp) else() message(FATAL_ERROR "Unrecognized backend: " ${USE_BACKEND}) @@ -535,6 +540,75 @@ elseif(USE_BACKEND STREQUAL "EIGEN") message(STATUS "Found Eigen3 at ${EIGEN3_INCLUDE_DIRS}") endif() endif() +elseif(USE_BACKEND STREQUAL "ONNX") + target_compile_definitions(katago PRIVATE USE_ONNX_BACKEND) + + # ONNX Runtime install tree (include/ lib/ bin/). The official prebuilt ORT packages do + # NOT ship the OpenVINO execution provider, so for Intel GPU acceleration ORT must be + # built from source with --use_openvino GPU (see the project's build notes). + set(ONNXRUNTIME_ROOT "" CACHE PATH "Path to ONNX Runtime package root (containing include/, lib/, bin/)") + if(NOT IS_DIRECTORY "${ONNXRUNTIME_ROOT}") + message(FATAL_ERROR "ONNXRUNTIME_ROOT does not exist: ${ONNXRUNTIME_ROOT}. Set -DONNXRUNTIME_ROOT=.") + endif() + set(ONNXRUNTIME_INCLUDE_DIR "${ONNXRUNTIME_ROOT}/include/onnxruntime") + if(NOT IS_DIRECTORY "${ONNXRUNTIME_INCLUDE_DIR}") + # Official prebuilt ORT packages (e.g. onnxruntime-win-x64-*.zip / -linux-x64-*.tgz) lay + # the headers flat under include/ (include/onnxruntime_cxx_api.h, ...), while an ORT + # built from source installs them under include/onnxruntime/. Support both layouts. + set(ONNXRUNTIME_INCLUDE_DIR "${ONNXRUNTIME_ROOT}/include") + endif() + if(NOT IS_DIRECTORY "${ONNXRUNTIME_INCLUDE_DIR}") + message(FATAL_ERROR "ONNX Runtime include directory not found under ${ONNXRUNTIME_ROOT}. Looked for both include/ and include/onnxruntime/.") + endif() + target_include_directories(katago SYSTEM PRIVATE "${ONNXRUNTIME_INCLUDE_DIR}") + if(WIN32) + set(ONNXRUNTIME_LIB "${ONNXRUNTIME_ROOT}/lib/onnxruntime.lib") + file(GLOB ONNXRUNTIME_DLLS "${ONNXRUNTIME_ROOT}/lib/*.dll" "${ONNXRUNTIME_ROOT}/bin/*.dll") + else() + find_library(ONNXRUNTIME_LIB onnxruntime HINTS "${ONNXRUNTIME_ROOT}/lib" "${ONNXRUNTIME_ROOT}/bin" "${ONNXRUNTIME_ROOT}") + endif() + if(NOT ONNXRUNTIME_LIB OR ONNXRUNTIME_LIB STREQUAL "ONNXRUNTIME_LIB-NOTFOUND" OR NOT EXISTS "${ONNXRUNTIME_LIB}") + message(FATAL_ERROR "Could not find onnxruntime library under ${ONNXRUNTIME_ROOT}. Looked for: ${ONNXRUNTIME_LIB}") + endif() + target_link_libraries(katago ${ONNXRUNTIME_LIB}) + + # The ONNX backend emits an ONNX ModelProto via the same OnnxModelBuilder as the + # TensorRT backend and hands the serialized bytes to Ort::Session. Generate onnx.pb.h + # from the vendored external/onnx/onnx.proto and link our own protobuf; the handoff to + # ORT is serialized bytes, so there is no ABI contact with whatever protobuf lives + # inside the ORT DLL. (Protobuf and protoc must be findable by find_package(Protobuf); + # for an ORT built from source these live under its _deps/protobuf-build.) + if(NOT DEFINED Protobuf_USE_STATIC_LIBS) + # The protobuf that ships with a from-source ONNX Runtime build (e.g. under its + # _deps/protobuf-build on Windows) is a static library. FindProtobuf otherwise treats + # libprotobuf.lib as a shared import lib on Windows and injects PROTOBUF_USE_DLLS into + # the imported target, which breaks linking against a static libprotobuf. Default to + # static, but let the user override (e.g. a shared protobuf from vcpkg). + set(Protobuf_USE_STATIC_LIBS TRUE) + endif() + find_package(Protobuf REQUIRED) + message(STATUS "Found Protobuf version: ${Protobuf_VERSION}") + set(ONNX_PROTO_DIR "${CMAKE_CURRENT_SOURCE_DIR}/external/onnx") + protobuf_generate_cpp(ONNX_PROTO_SRCS ONNX_PROTO_HDRS "${ONNX_PROTO_DIR}/onnx.proto") + set_source_files_properties(${ONNX_PROTO_SRCS} PROPERTIES COMPILE_OPTIONS "-w") + target_sources(katago PRIVATE ${ONNX_PROTO_SRCS} neuralnet/onnxmodelbuilder.cpp) + target_include_directories(katago SYSTEM PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${Protobuf_INCLUDE_DIRS}) + # Link the imported target rather than ${Protobuf_LIBRARIES}: when protobuf comes from a + # CMake package config (e.g. vcpkg), the variable can resolve to the DLL itself rather + # than the import lib, and it also omits protobuf's own dependencies such as abseil. + target_link_libraries(katago protobuf::libprotobuf) + + # Deploy the ORT runtime DLLs next to katago.exe so the build dir is self-contained. + # NOTE: OpenVINO's own runtime DLLs are not shipped by ORT and must be copied + # separately (see the project's build notes). + if(WIN32 AND ONNXRUNTIME_DLLS) + foreach(_onnxruntime_dll IN LISTS ONNXRUNTIME_DLLS) + add_custom_command(TARGET katago POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${_onnxruntime_dll}" + $) + endforeach() + endif() endif() if(USE_BIGGER_BOARDS_EXPENSIVE) diff --git a/cpp/configs/gtp_example.cfg b/cpp/configs/gtp_example.cfg index 7c3a8f8341..d464bde47a 100644 --- a/cpp/configs/gtp_example.cfg +++ b/cpp/configs/gtp_example.cfg @@ -462,6 +462,86 @@ searchFactorWhenWinningThreshold = 0.95 # "auto" (default) uses the GEMM only in FP16, where it is slightly faster. # cudaUse1x1Matmul = auto +# ------------------------------ +# ONNX Runtime backend settings +# ------------------------------ +# These only apply when using the ONNX version of KataGo (USE_BACKEND=ONNX). +# The official prebuilt ONNX Runtime packages do NOT include the OpenVINO +# execution provider; for Intel GPU (Arc) acceleration, build ORT from source +# with --use_openvino GPU. + +# Execution provider. One of (see Compiling.md "Execution provider support matrix" +# for verification status and build requirements): +# cpu (default) - experimental, best-effort +# openvino - verified upstream (Windows + Intel GPU) +# cuda / tensorrt / migraphx / coreml - experimental, NOT tested upstream. +# These require an ONNX Runtime built with the matching EP plus its runtime deps; +# "coreml" additionally needs build work upstream (macOS). +# Use "openvino" for Intel Arc/iGPU/NPU. +# onnxProvider = cpu + +# Provider-specific device selection (mostly for cuda / tensorrt / migraphx). +# For the OpenVINO EP, a nonzero device index is appended to device_type as an OpenVINO +# device suffix (e.g. onnxDeviceToUse = 1 with device_type GPU selects "GPU.1"). Prefer +# the per-thread device_type overrides below for mixed CPU/GPU/NPU setups. +# +# NOTE: the index suffix is only appended when device_type is a simple device name. +# It is silently ignored (with a warning) for composite/qualified device strings such as +# AUTO:GPU,CPU or MULTI:GPU.0,GPU.1 - pick the device explicitly there via the +# onnxOpenVINODeviceTypeThread overrides instead. Use only one of the two mechanisms +# per thread: onnxDeviceToUse* selects by numeric index, onnxOpenVINODeviceTypeThread +# selects by full device string. +# onnxDeviceToUse = 0 +# onnxDeviceToUseThread0 = 0 +# onnxDeviceToUseThread1 = 1 + +# OpenVINO EP options (only used when onnxProvider = openvino): +# Device type: GPU, CPU, NPU, AUTO:GPU,CPU, MULTI:GPU.0,GPU.1, etc. +# onnxOpenVINODeviceType = GPU + +# Per-thread device type assignment (optional). +# Overrides onnxOpenVINODeviceType for the specified thread. +# This allows mixing CPU, GPU, and NPU inference within the same process. +# onnxOpenVINODeviceTypeThread0 = NPU +# onnxOpenVINODeviceTypeThread1 = GPU.0 +# onnxOpenVINODeviceTypeThread2 = GPU.1 +# onnxOpenVINODeviceTypeThread3 = CPU + +# OpenVINO EP: cache compiled graphs under cwd to skip recompile on restart; unset => full recompile every startup +# onnxOpenVINOCacheDir = katago_ov_cache +# Optional precision override: FP16, FP32, ACCURACY +# onnxOpenVINOPrecision = FP16 +# +# NOTE: the ONNX backend does not support the global useFP16 flag used by other backends. +# Setting "useFP16 = true" fails with an error, since ONNX Runtime execution providers decide +# inference precision themselves. Use onnxOpenVINOPrecision above to control precision instead, +# or leave useFP16 unset/auto. + +# Skip the scale8 FP16-range workaround (default false = apply it). scale8 keeps +# convnet activations 8x smaller so they stay inside the FP16 range OpenVINO infers +# in; the cost is MISH_SCALE8 subgraphs that block OpenVINO's fused-Mish (~2x slower +# on large-board convnets). Keep off (default); set true only for FP32 precision or +# small-board/transformer workloads where FP16 overflow is not a practical risk. +# onnxSkipScale8 = false + +# Optional OpenVINO execution streams / inference threads / priority: +# onnxOpenVINONumStreams = 1 +# onnxOpenVINONumOfThreads = 1 +# onnxOpenVINOModelPriority = DEFAULT + +# Per-device-type EP option overrides (optional). +# Fine-tune streams, precision etc. per device type (NPU, GPU, CPU). +# "GPU" matches GPU, GPU.0, GPU.1 and other GPU variants. +# onnxOpenVINODeviceConfig_NPU_NumStreams = 4 +# onnxOpenVINODeviceConfig_NPU_Precision = FP16 +# onnxOpenVINODeviceConfig_GPU_NumStreams = 2 +# onnxOpenVINODeviceConfig_CPU_NumOfThreads = 2 + +# Run the trunk block stack channel-last (NHWC) for transformer models. +# Default true (NHWC, matching the TensorRT backend). Only takes effect for +# models with transformer blocks; convnets ignore this. +# onnxTransformerNHWC = true + # ------------------------------ # Metal GPU settings # ------------------------------ diff --git a/cpp/main.cpp b/cpp/main.cpp index f9c95e09a6..9fd5eee979 100644 --- a/cpp/main.cpp +++ b/cpp/main.cpp @@ -253,6 +253,8 @@ string Version::getKataGoVersionFullInfo() { out << "Using OpenCL backend" << endl; #elif defined(USE_EIGEN_BACKEND) out << "Using Eigen(CPU) backend" << endl; +#elif defined(USE_ONNX_BACKEND) + out << "Using ONNX Runtime backend" << endl; #else out << "Using dummy backend" << endl; #endif @@ -289,6 +291,8 @@ string Version::getGitRevisionWithBackend() { s += "-opencl"; #elif defined(USE_EIGEN_BACKEND) s += "-eigen"; +#elif defined(USE_ONNX_BACKEND) + s += "-onnx"; #else s += "-dummy"; #endif diff --git a/cpp/neuralnet/onnxbackend.cpp b/cpp/neuralnet/onnxbackend.cpp new file mode 100644 index 0000000000..468c966361 --- /dev/null +++ b/cpp/neuralnet/onnxbackend.cpp @@ -0,0 +1,1028 @@ +// ONNX Runtime backend for KataGo. +// +// Loads standard .bin.gz KataGo model files, converts the ModelDesc to a serialized +// ONNX ModelProto via the same OnnxModelBuilder that the TensorRT backend uses, and +// hands the bytes to an Ort::Session. Inference is run through ONNX Runtime with a +// configurable execution provider (CPU, OpenVINO, CUDA, TensorRT, MIGraphX, CoreML) +// selected at runtime via the onnxProvider config key. +// Only the OpenVINO provider is verified upstream (see Compiling.md "Execution +// provider support matrix"); the others are experimental code paths that require a +// provider-enabled ONNX Runtime build. +// +// The IO tensor protocol is identical to the TensorRT ONNX-emitter path (see +// onnxmodelbuilder.h): four NCHW float32 inputs InputMask / InputSpatial / +// InputGlobal / InputMeta and five NCHW float32 outputs OutputPolicyPass / +// OutputPolicy / OutputValue / OutputScoreValue / OutputOwnership, all raw logits. +// The C++ getOutput below reproduces the TensorRT backend's post-processing exactly +// (per-row optimism blend, inverse-symmetry, version-branched score-value decode) so +// that the same downstream decode path is shared. + +#ifdef USE_ONNX_BACKEND + +#include "../neuralnet/nninterface.h" +#include "../neuralnet/nneval.h" +#include "../neuralnet/nninputs.h" +#include "../neuralnet/modelversion.h" +#include "../neuralnet/onnxmodelbuilder.h" + +#include +#ifdef __APPLE__ +#include +#endif +#ifdef _WIN32 +// dml_provider_factory.h is only shipped by DirectML-enabled ONNX Runtime packages (e.g. +// Microsoft.ML.OnnxRuntime.DirectML); the stock CPU prebuilt does not include it. Guard on +// availability so builds against such an ORT still compile - the DirectML provider then +// fails at runtime with a clear error instead of at compile time. +#if __has_include() +#include +#define KATAGO_ONNX_HAS_DML_PROVIDER_FACTORY 1 +#endif +#endif + +#include +#include +#include +#include + +using namespace std; + +//-------------------------------------------------------------- + +// ONNX execution providers this backend knows how to wire up. This list is the wiring +// surface, not a statement of support: only the OpenVINO provider is verified upstream +// (see Compiling.md "Execution provider support matrix"). Exposing a new EP = add its +// name here plus an AppendExecutionProvider_* branch in ComputeHandle. +static const char* const kKnownProviders[] = { + "cpu", "openvino", "cuda", "tensorrt", "migraphx", "coreml", "directml", +}; + +//-------------------------------------------------------------- + +struct LoadedModel { + ModelDesc modelDesc; + // One-time scale8 transform (see maybeApplyScale8). All server threads share this + // LoadedModel, so whichever compute handle is created first decides for everyone. + // + // scale8Resolved is only ever accessed under scale8Mutex, so a plain bool suffices. + // The mutex also establishes the happens-before between the write to modelDesc here and + // the subsequent unsynchronized reads in OnnxModelBuilder::build() of every thread: + // each thread runs maybeApplyScale8 (under the lock) before building its graph. + mutable bool scale8Resolved; + mutable std::mutex scale8Mutex; + + LoadedModel(const string& fileName, const string& expectedSha256) { + if(Global::isSuffix(fileName, ".onnx")) + throw StringError( + "ONNX backend: loading a raw .onnx file is not supported by this backend. " + "Feed a standard KataGo .bin.gz model instead (this backend builds the ONNX " + "graph from the model weights internally)."); + ModelDesc::loadFromFileMaybeGZipped(fileName, modelDesc, expectedSha256); + scale8Resolved = false; + } + + // Apply the scale8 FP16-range workaround exactly once per model, unless skipped via + // onnxSkipScale8. Must run before any ComputeHandle builds the graph from modelDesc. + void maybeApplyScale8(bool skip) const { + std::lock_guard lock(scale8Mutex); + if(!scale8Resolved) { + if(!skip) + const_cast(this)->modelDesc.applyScale8ToReduceActivations(); + scale8Resolved = true; + } + } + + LoadedModel() = delete; + LoadedModel(const LoadedModel&) = delete; + LoadedModel& operator=(const LoadedModel&) = delete; +}; + +LoadedModel* NeuralNet::loadModelFile(const string& file, const string& expectedSha256) { + return new LoadedModel(file, expectedSha256); +} + +void NeuralNet::freeLoadedModel(LoadedModel* loadedModel) { + delete loadedModel; +} + +const ModelDesc& NeuralNet::getModelDesc(const LoadedModel* loadedModel) { + return loadedModel->modelDesc; +} + +//-------------------------------------------------------------- + +struct ComputeContext { + Ort::Env env; + int nnXLen; + int nnYLen; + string providerName; + string openvinoDeviceType; + string openvinoCacheDir; + // Optional OpenVINO provider options (empty = not passed to ORT) + string openvinoPrecision; // FP16 / FP32 / ACCURACY + string openvinoNumStreams; // 1-8 + string openvinoNumOfThreads; // positive int (infer requests per session) + string openvinoModelPriority; // LOW / MEDIUM / HIGH / DEFAULT + bool transformerNHWC; // run the trunk block stack channel-last (NHWC) + bool skipScale8; // skip the scale8 FP16-range workaround (see createComputeContext) + + // Per-thread device type (index = serverThreadIdx). Filled with openvinoDeviceType + // by default; individual entries are replaced by onnxOpenVINODeviceTypeThread. + std::vector perThreadDeviceType; + + // Per-device-type EP option overrides. + // Outer key = short device name ("NPU", "GPU", "CPU"). + // Inner key = ORT EP option key ("num_streams", "precision", ...). + std::unordered_map> deviceConfigOverrides; + + ComputeContext(int xLen, int yLen) + : env(ORT_LOGGING_LEVEL_WARNING, "KataGoOnnx"), + nnXLen(xLen), + nnYLen(yLen), + providerName("cpu"), + openvinoDeviceType("GPU"), + openvinoCacheDir(""), + openvinoPrecision(""), + openvinoNumStreams(""), + openvinoNumOfThreads(""), + openvinoModelPriority(""), + transformerNHWC(true), + skipScale8(false) + {} +}; + +ComputeContext* NeuralNet::createComputeContext( + const std::vector& gpuIdxs, + Logger* logger, + int nnXLen, + int nnYLen, + const string& homeDataDirOverride, + enabled_t useFP16Mode, + const LoadedModel* loadedModel, + ConfigParser& cfg +) { + (void)gpuIdxs; + (void)homeDataDirOverride; + (void)loadedModel; + // The emitted ONNX graph is fp32; inference precision is chosen internally by the execution + // provider (e.g. OpenVINO downcasts to FP16 per onnxOpenVINOPrecision). KataGo's global useFP16 + // flag therefore cannot be honored here - fail loudly instead of silently ignoring a request. + if(useFP16Mode == enabled_t::True) + throw StringError( + "ONNX backend: the global useFP16 flag is not supported and cannot be honored. " + "Precision is controlled by the execution provider; for the OpenVINO provider set " + "onnxOpenVINOPrecision (e.g. FP16/FP32/ACCURACY). Leave useFP16 unset or set it to false/auto."); + + ComputeContext* ctx = new ComputeContext(nnXLen, nnYLen); + + // Provider selection. Default CPU; OpenVINO is the EP used for Intel Arc GPUs. + string providerName = cfg.contains("onnxProvider") ? cfg.getString("onnxProvider") : "cpu"; + ctx->providerName = Global::toLower(providerName); + + // OpenVINO EP options. + ctx->openvinoDeviceType = cfg.contains("onnxOpenVINODeviceType") ? cfg.getString("onnxOpenVINODeviceType") : "GPU"; + ctx->openvinoCacheDir = cfg.contains("onnxOpenVINOCacheDir") ? cfg.getString("onnxOpenVINOCacheDir") : ""; + ctx->openvinoPrecision = cfg.contains("onnxOpenVINOPrecision") ? cfg.getString("onnxOpenVINOPrecision") : ""; + ctx->openvinoNumStreams = cfg.contains("onnxOpenVINONumStreams") ? cfg.getString("onnxOpenVINONumStreams") : ""; + ctx->openvinoNumOfThreads = cfg.contains("onnxOpenVINONumOfThreads") ? cfg.getString("onnxOpenVINONumOfThreads") : ""; + ctx->openvinoModelPriority = cfg.contains("onnxOpenVINOModelPriority") ? cfg.getString("onnxOpenVINOModelPriority") : ""; + + // Trunk layout for transformer models. Default NHWC (channel-last), matching the TensorRT + // backend's trtTransformerNHWC default; NHWC is markedly faster for transformer trunks on + // OpenVINO GPU/NPU and ignored entirely for models without transformer blocks. + ctx->transformerNHWC = cfg.contains("onnxTransformerNHWC") ? cfg.getBool("onnxTransformerNHWC") : true; + + // Skip the scale8 FP16-range workaround (default false = apply it). scale8 keeps + // convnet activations 8x smaller so they stay inside the FP16 range OpenVINO infers + // in; the cost is MISH_SCALE8 subgraphs that block OpenVINO's fused-Mish (~2x slower + // on large-board convnets). Keep on (default); set true only for FP32 precision or + // small-board/transformer workloads where FP16 overflow is not a practical risk. + ctx->skipScale8 = cfg.contains("onnxSkipScale8") ? cfg.getBool("onnxSkipScale8") : false; + + // --- Per-thread device type assignment --- + // Pre-parse onnxOpenVINODeviceTypeThread keys so ComputeHandle can look up + // the device type for each server thread without reaching back into ConfigParser. + { + int numThreads = 1; + if(cfg.contains("numNNServerThreadsPerModel")) + numThreads = cfg.getInt("numNNServerThreadsPerModel", 1, 1024); + ctx->perThreadDeviceType.resize(numThreads, ctx->openvinoDeviceType); + for(int t = 0; t < numThreads; t++) { + string key = "onnxOpenVINODeviceTypeThread" + Global::intToString(t); + if(cfg.contains(key)) + ctx->perThreadDeviceType[t] = cfg.getString(key); + } + } + + // --- Per-device-type EP option overrides --- + // onnxOpenVINODeviceConfig__ = value + // e.g. onnxOpenVINODeviceConfig_NPU_NumStreams = 4 + // maps to deviceConfigOverrides["NPU"]["num_streams"] = "4" + { + static const char* knownDevices[] = {"NPU", "GPU", "CPU"}; + struct OptMapping { const char* cfgSuffix; const char* ortKey; }; + static const OptMapping epOptMappings[] = { + {"NumStreams", "num_streams"}, + {"Precision", "precision"}, + {"NumOfThreads", "num_of_threads"}, + {"ModelPriority", "model_priority"}, + {"CacheDir", "cache_dir"}, + }; + for(const char* dev : knownDevices) { + string devPrefix = string("onnxOpenVINODeviceConfig_") + dev + "_"; + for(const auto& m : epOptMappings) { + string key = devPrefix + m.cfgSuffix; + if(cfg.contains(key)) + ctx->deviceConfigOverrides[dev][m.ortKey] = cfg.getString(key); + } + } + } + + { + bool knownProvider = false; + for(const char* p : kKnownProviders) { + if(ctx->providerName == p) { + knownProvider = true; + break; + } + } + if(!knownProvider) + throw StringError( + "ONNX backend: unknown onnxProvider '" + ctx->providerName + + "'. Known providers: cpu, openvino, cuda, tensorrt, migraphx, coreml, directml " + "(verification status and build requirements: see Compiling.md " + "'Execution provider support matrix')."); + } + + if(logger != NULL) + logger->write("ONNX backend: creating compute context for " + + Global::intToString(nnXLen) + "x" + Global::intToString(nnYLen) + + " with provider '" + ctx->providerName + "'"); + + return ctx; +} + +void NeuralNet::freeComputeContext(ComputeContext* computeContext) { + delete computeContext; +} + +//-------------------------------------------------------------- +// Helper: extract a short device name from an OpenVINO device_type +// string for matching onnxOpenVINODeviceConfig__