diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml index ee55b6164..3bb8707a5 100644 --- a/.github/workflows/gpu-tests.yml +++ b/.github/workflows/gpu-tests.yml @@ -232,8 +232,9 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" echo "instance $IID ready — proceeding to tests" exit 0 + else + rc=$? fi - rc=$? echo "::warning::instance $IID not ready (rc=$rc: 1=timeout, 2=image/scheduling failure); destroying and trying another host" destroy "$IID"; rm -f "$RUNNER_TEMP/vast_instance_id" done @@ -267,7 +268,10 @@ jobs: # so we test exactly what will land. workflow_dispatch: the chosen branch ref. REF: ${{ github.ref }} run: | - SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST" + # ServerAlive*: ConnectTimeout only covers connection setup; without keepalives a box + # that wedges or drops off the network mid-suite would hang this step silently until + # the 240-minute job timeout. 60s x 10 fails the run ~10 minutes after the box goes dark. + SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o ServerAliveInterval=60 -o ServerAliveCountMax=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST" # Defense-in-depth: never interpolate an unvalidated ref into the remote `bash -lc`. case "$REF" in ''|*[!A-Za-z0-9._/-]*) echo "::error::invalid ref: '$REF'"; exit 1 ;; @@ -327,10 +331,21 @@ jobs: echo; echo "No individual test failures parsed (build/infra error?). Last lines:" echo '```'; tail -n 40 "$OUT" 2>/dev/null || echo "(no output captured)"; echo '```' fi - echo; echo "Full output is in the \"Run GPU tests\" step log." + echo; echo "Full output: \"Run GPU tests\" step log, or the \`gpu-test-log\` artifact (survives UI log truncation)." fi } >> "$GITHUB_STEP_SUMMARY" + # The step log gets truncated/rotated in the UI for multi-hour runs (see the machine-info + # comment above); the artifact keeps the complete output retrievable. + - name: Upload full test log + if: always() && (steps.tests.outcome == 'success' || steps.tests.outcome == 'failure') + uses: actions/upload-artifact@v4 + with: + name: gpu-test-log + path: ${{ runner.temp }}/gpu_test_out.txt + if-no-files-found: ignore + retention-days: 14 + # --- Teardown: ALWAYS destroy the instance (cost guardrail) --- - name: Destroy instance if: always() diff --git a/Makefile b/Makefile index 001cc66f1..be6d53811 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-shared \ clean-recursion-elfs clean test test-asm \ test-rust test-executor test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \ -test-fast test-prover test-prover-all test-disk-spill test-math-cuda test-cuda-integration test-cuda-fallback \ +test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-cuda-integration test-cuda-fallback \ test-prover-cuda test-prover-comprehensive-cuda \ bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \ update-ethrex-fixture-checksums check-ethrex-fixture-checksums @@ -288,7 +288,7 @@ test-math-cuda: cargo test -p math-cuda --release # End-to-end cuda dispatch coverage (requires NVIDIA GPU + nvcc). -# Asserts every R1/R2/R3 GPU counter fired on a real prove. +# Asserts the R1-R4 GPU dispatch counters fired on a real prove. test-cuda-integration: cargo test -p lambda-vm-prover --release --features cuda \ --test cuda_path_integration -- --ignored --nocapture @@ -308,7 +308,8 @@ test-prover-cuda: --features lambda-vm-prover/cuda -- --test-threads=1 # The comprehensive all-instructions prove (ignored by default) on the GPU path (requires -# NVIDIA GPU + nvcc). GPU counterpart of CPU CI's merge-queue-only comprehensive job. +# NVIDIA GPU + nvcc). GPU counterpart of the all-instructions half of CPU CI's merge-queue-only +# comprehensive job (the CPU job also runs test_recursion_execute; recursion has no GPU leg yet). test-prover-comprehensive-cuda: cargo test --release -p lambda-vm-prover --features cuda \ test_prove_elfs_all_instructions_64_full -- --ignored --test-threads=1 --nocapture diff --git a/docs/roadmap.md b/docs/roadmap.md index 3658a946b..97ffa3138 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -55,8 +55,8 @@ The first version is going to use the primitives contained in [lambdaworks](http | Feature | Description | Status | |---------------------------- |-----------------------------------|--------------| | Fields | Improve field performance using assembly | Planned | -| GPU-Fast-Fourier transform | Implement GPU version of FFT | Planned | -| GPU-Merkle tree | Implement GPU version for Merkle trees | Planned | +| GPU-Fast-Fourier transform | Implement GPU version of FFT | Done | +| GPU-Merkle tree | Implement GPU version for Merkle trees | Done | | Parallel trace generation | Use GPU for fast trace generation | Planned | -| GPU-FRI | Perform FRI on GPU | Planned | +| GPU-FRI | Perform FRI on GPU | Done | \ No newline at end of file diff --git a/prover/src/tests/prove_elfs_tests.rs b/prover/src/tests/prove_elfs_tests.rs index 2b8c16342..527c092c5 100644 --- a/prover/src/tests/prove_elfs_tests.rs +++ b/prover/src/tests/prove_elfs_tests.rs @@ -1440,7 +1440,7 @@ fn test_verify_rejects_tampered_public_output() { /// - Division: DIV, DIVU, REM, REMU /// - Control: LUI, AUIPC, JALR #[test] -#[ignore] // Slow: run with `cargo test --ignored` or `make test-prover-all` +#[ignore] // Slow: run with `cargo test -- --ignored` or `make test-prover-all` fn test_prove_elfs_all_instructions_64_full() { let _ = env_logger::builder().is_test(true).try_init(); diff --git a/scripts/gpu_test.sh b/scripts/gpu_test.sh index 70371395f..661339f11 100755 --- a/scripts/gpu_test.sh +++ b/scripts/gpu_test.sh @@ -44,6 +44,17 @@ nvidia-smi --query-gpu=name,driver_version,compute_cap --format=csv,noheader # conservative CUDA version binds a known driver-symbol set instead. (This is cudarc's # host-side driver-API floor — independent of the PTX/driver version the offer filter targets.) log "pinning cudarc to $CUDARC_PIN" +# Guard the sed anchors: if math-cuda's cudarc features are ever renamed/reformatted, a silent +# no-op here would bring the fallback-latest driver-symbol panic back with a confusing signature. +for anchor in '"cuda-version-from-build-system"' '"fallback-latest"'; do + grep -qF "$anchor" crypto/math-cuda/Cargo.toml \ + || { echo "ERROR: sed anchor $anchor not found in crypto/math-cuda/Cargo.toml — update this script's cudarc pin" >&2; exit 1; } +done +# Restore the tracked file on exit so a manual run on a dev box doesn't leave the tree dirty +# (CI doesn't need this — the workflow re-checks-out before every run — but it's harmless there). +CUDARC_TOML_BACKUP="$(mktemp)" +cp crypto/math-cuda/Cargo.toml "$CUDARC_TOML_BACKUP" +trap 'cp "$CUDARC_TOML_BACKUP" crypto/math-cuda/Cargo.toml; rm -f "$CUDARC_TOML_BACKUP"' EXIT sed -i "s/\"cuda-version-from-build-system\"/\"${CUDARC_PIN}\"/; /\"fallback-latest\"/d" \ crypto/math-cuda/Cargo.toml