Skip to content

Share the Rust micro-benchmark workflow #8777

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
connortsui20 merged 1 commit into from
Jul 15, 2026
Merged

Share the Rust micro-benchmark workflow #8777

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 204 additions & 0 deletions .github/workflows/bench-core.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
name: Rust micro-benchmarks

# Reusable workflow for the Rust micro-benchmarks (random access, compression). Called by bench.yml
# (mode=develop) and bench-pr.yml (mode=pr). The matrix is resolved declaratively from
# bench-orchestrator; see bench-orchestrator/bench_orchestrator/matrix.py.

on:
workflow_call:
inputs:
mode:
required: true
type: string
profile:
required: false
type: string
default: "micro"

jobs:
plan:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
matrix: ${{ steps.resolve.outputs.matrix }}
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
if: inputs.mode == 'pr'
with:
ref: ${{ github.event.pull_request.head.sha }}
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
if: inputs.mode != 'pr'
- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false
- name: Resolve benchmark matrix
id: resolve
shell: bash
env:
PROFILE: ${{ inputs.profile }}
run: |
echo "matrix=$(uv run --project bench-orchestrator vx-bench matrix "$PROFILE")" >> "$GITHUB_OUTPUT"

bench:
needs: plan
timeout-minutes: 120
env:
RUST_BACKTRACE: full
VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.plan.outputs.matrix) }}

runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.id, (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && '/extras=s3-cache' || '')
|| 'ubuntu-latest' }}
steps:
- uses: runs-on/action@v2
if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
with:
sccache: s3
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
if: inputs.mode == 'pr'
with:
ref: ${{ github.event.pull_request.head.sha }}

- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
if: inputs.mode != 'pr'
- name: Setup benchmark environment
run: sudo bash scripts/setup-benchmark.sh
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
enable-sccache: ${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}
- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false

- name: Install DuckDB
run: |
wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb
chmod +x duckdb
echo "$PWD" >> "$GITHUB_PATH"

- uses: ./.github/actions/system-info

- name: Build binary
shell: bash
env:
RUSTFLAGS: "-C target-cpu=native"
run: |
cargo build --bin ${{ matrix.id }} --profile release_debug ${{ matrix.build_args }} --features unstable_encodings

- name: Setup Polar Signals
if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1
with:
polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}"
project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
profiling_frequency: 199
extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz

- name: Run ${{ matrix.name }} benchmark (per-combination)
if: ${{ matrix.split }}
shell: bash
run: |
python3 scripts/random-access-split.py ${{ inputs.mode != 'pr' && '--v3' || '' }}

- name: Run ${{ matrix.name }} benchmark
if: ${{ !matrix.split }}
shell: bash
run: |
bash scripts/bench-taskset.sh target/release_debug/${{ matrix.id }} \
--formats ${{ matrix.formats }} \
-d gh-json \
-o results.json \
${{ inputs.mode != 'pr' && '--gh-json-v3 results.v3.jsonl' || '' }}

- name: Setup AWS CLI
if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
with:
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
aws-region: us-east-1

- name: Compare results
if: inputs.mode == 'pr'
shell: bash
run: |
set -Eeu -o pipefail -x

python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
gzip -d -c data.json.gz > base.json

echo "# Benchmarks: ${{ matrix.name }}" > comment.md
echo '' >> comment.md
uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \
>> comment.md
cat comment.md >> "$GITHUB_STEP_SUMMARY"

- name: Comment PR
if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3
with:
file-path: comment.md
comment-tag: bench-pr-comment-${{ matrix.id }}

- name: Comment PR on failure
if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3
with:
message: |
# BENCHMARK FAILED

Benchmark `${{ matrix.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
comment-tag: bench-pr-comment-${{ matrix.id }}

- name: Upload Benchmark Results
if: inputs.mode == 'develop'
shell: bash
run: |
bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json

# v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live benchmarks
# website (see bench.yml for the full rationale). Gated on inputs.mode == 'develop' + the
# ingest-role ARN var. post-ingest.py mints the RDS IAM token (boto3) from the assumed
# GitHubBenchmarkIngestRole; sslmode=verify-full validates the cert.
- name: Configure AWS credentials for v4 ingest (OIDC)
if: inputs.mode == 'develop' && vars.GH_BENCH_INGEST_ROLE_ARN != ''
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
with:
role-to-assume: ${{ vars.GH_BENCH_INGEST_ROLE_ARN }}
aws-region: ${{ vars.RDS_BENCH_REGION }}
- name: Ingest results to v4 Postgres
if: inputs.mode == 'develop' && vars.GH_BENCH_INGEST_ROLE_ARN != ''
shell: bash
env:
RDS_BENCH_INSTANCE_ENDPOINT: ${{ vars.RDS_BENCH_INSTANCE_ENDPOINT }}
RDS_BENCH_DB_NAME: ${{ vars.RDS_BENCH_DB_NAME }}
AWS_REGION: ${{ vars.RDS_BENCH_REGION }}
BENCH_SITE_BASE_URL: ${{ vars.BENCH_SITE_BASE_URL }}
BENCH_REVALIDATE_TOKEN: ${{ secrets.BENCH_REVALIDATE_TOKEN }}
run: |
set -Eeuo pipefail
curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
-o "${RUNNER_TEMP}/rds-global-bundle.pem"
DSN="postgresql://bench_ingest@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
uv run --no-project --with 'psycopg[binary]' --with boto3 --with xxhash \
scripts/post-ingest.py results.v3.jsonl \
--postgres "${DSN}" \
--commit-sha "${{ github.sha }}" \
--region "${AWS_REGION}"

- name: Alert incident.io
if: failure() && inputs.mode == 'develop'
uses: ./.github/actions/alert-incident-io
with:
api-key: ${{ secrets.INCIDENT_IO_ALERT_TOKEN }}
alert-title: "${{ matrix.name }} benchmark failed on develop"
deduplication-key: ci-bench-${{ matrix.id }}-failure
121 changes: 6 additions & 115 deletions .github/workflows/bench-pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,122 +19,13 @@ permissions:
id-token: write # enables AWS-GitHub OIDC

jobs:
# Rust micro-benchmarks (random access, compression). Matrix resolved from the `micro` profile.
bench:
timeout-minutes: 120
runs-on: >-
${{ github.repository == 'vortex-data/vortex'
&& format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
|| 'ubuntu-latest' }}
strategy:
matrix:
benchmark:
- id: random-access-bench
name: Random Access
build_args: "--features lance"
- id: compress-bench
name: Compression
steps:
- uses: runs-on/action@v2
if: github.event.pull_request.head.repo.fork == false
with:
sccache: s3
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
ref: ${{ github.event.pull_request.head.sha }}
- name: Setup benchmark environment
run: sudo bash scripts/setup-benchmark.sh
- uses: ./.github/actions/setup-rust
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
enable-sccache: ${{ github.event.pull_request.head.repo.fork == false && 'true' || 'false' }}

- name: Install DuckDB
run: |
wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.4/duckdb_cli-linux-amd64.zip | funzip > duckdb
chmod +x duckdb
echo "$PWD" >> $GITHUB_PATH

- uses: ./.github/actions/system-info

- name: Build binary
shell: bash
env:
RUSTFLAGS: "-C target-cpu=native"
run: |
cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings

- name: Setup Polar Signals
if: github.event.pull_request.head.repo.fork == false
uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1
with:
polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}"
project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
profiling_frequency: 199
extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz

- name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
if: matrix.benchmark.id == 'random-access-bench'
shell: bash
env:
RUST_BACKTRACE: full
VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
run: |
python3 scripts/random-access-split.py

- name: Run ${{ matrix.benchmark.name }} benchmark
if: matrix.benchmark.id != 'random-access-bench'
shell: bash
env:
RUST_BACKTRACE: full
VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
run: |
bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json

- name: Setup AWS CLI
if: github.event.pull_request.head.repo.fork == false
uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6
with:
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
aws-region: us-east-1

- name: Install uv
uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
with:
sync: false

- name: Compare results
shell: bash
run: |
set -Eeu -o pipefail -x

python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
gzip -d -c data.json.gz > base.json

echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md
echo '' >> comment.md
uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
>> comment.md
cat comment.md >> $GITHUB_STEP_SUMMARY

- name: Comment PR
if: github.event.pull_request.head.repo.fork == false
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3
with:
file-path: comment.md
comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}

- name: Comment PR on failure
if: failure() && github.event.pull_request.head.repo.fork == false
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3
with:
message: |
# BENCHMARK FAILED

Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
uses: ./.github/workflows/bench-core.yml
secrets: inherit
with:
mode: "pr"
profile: "micro"

sql:
uses: ./.github/workflows/sql-benchmarks.yml
Expand Down
Loading
Loading