release-performance.yml

name: Release Performance
run-name: >-
  ${{ github.event.pull_request.title || github.event.head_commit.message || github.workflow }}

# Every push to main compares main with the latest release on the same runner. A regression fails
# the run and opens or updates one issue labelled performance-regression; it does not gate the
# release pull request. Pull requests that change the comparison itself run it too.
on:
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      baseline:
        description: "Published sqlbuild version to compare against (default: the release before the candidate)"
        required: false
        type: string
  pull_request:
    paths:
      - .github/workflows/release-performance.yml
      - scripts/compare_release_performance.py
      - scripts/release_performance/**
      - scripts/compile_performance_ratio/_helpers/edit.py
      - scripts/compile_performance_ratio/_helpers/measure.py
      - scripts/compile_performance_ratio/_helpers/report.py
      - scripts/compile_performance_ratio/constants.py
      - scripts/compile_performance_ratio/exceptions.py
      - scripts/compile_performance_ratio/models.py
      - scripts/cold_compile_performance/_helpers/dense_project.py
      - scripts/cold_compile_performance/constants.py

permissions:
  contents: read

# Every main commit gets its own comparison, so a later push never cancels the run that would
# report an earlier regression. Pull request runs supersede older runs of the same pull request.
concurrency:
  group: >-
    release-performance-${{ github.event_name }}-${{
    github.event_name == 'push' && github.sha || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  compare:
    name: Compare with the previous release
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    outputs:
      regressed: ${{ steps.compare.outputs.regressed }}
    steps:
      - uses: actions/checkout@v4
        with:
          # Release tags identify the previous release before its wheels reach PyPI.
          fetch-depth: 0
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.3"
          enable-cache: true
          cache-dependency-glob: |
            uv.lock
            Cargo.lock
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
      - name: Install Python
        run: uv python install 3.12
      - name: Install dependencies
        run: uv sync --locked --all-groups --all-extras
      - name: Build candidate wheel
        run: |
          uv build --wheel --out-dir "${RUNNER_TEMP}/candidate"
          uv build --wheel crates/sqlbuild-cli --out-dir "${RUNNER_TEMP}/candidate"
      # A release tag created while the candidate built names the baseline's benchmark generator.
      - name: Fetch release tags
        run: git fetch --tags --force --quiet origin
      # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited
      # model recompiles uncached; tighten it back to 1.15x once native rendering is the default.
      - name: Compare candidate and previous release on this runner
        id: compare
        env:
          BASELINE: ${{ inputs.baseline }}
          EVENT_NAME: ${{ github.event_name }}
        run: |
          # A main commit that is not itself a release still carries the latest release's version,
          # so it is compared with that release rather than the one before it. A release commit
          # bumps the version and is compared with the release before it.
          if [[ "${EVENT_NAME}" == "push" && -z "${BASELINE}" ]]; then
            project_version() {
              python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])'
            }
            version="$(project_version < pyproject.toml)"
            previous="$(git show HEAD^:pyproject.toml | project_version)" || previous=""
            if [[ "${version}" == "${previous}" ]]; then
              BASELINE="${version}"
            fi
          fi
          wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl)
          evidence="${RUNNER_TEMP}/release-performance"
          mkdir -p "${evidence}"
          args=(
            --candidate "${wheels[0]}"
            --work-dir "${RUNNER_TEMP}/release-performance-work"
            --output "${evidence}/release-performance.json"
          )
          if [[ -n "${BASELINE}" ]]; then
            args+=(--baseline "${BASELINE}")
          fi
          # The comparison's own exit status, not tee's, decides the outcome.
          set +e
          uv run python -m scripts.compare_release_performance "${args[@]}" \
            | tee "${evidence}/release-performance.md"
          status="${PIPESTATUS[0]}"
          set -e
          regressed=false
          if [[ "${status}" != "0" && -f "${evidence}/release-performance.json" ]] && \
             jq -e 'any(.verdicts[]; .regressed)' "${evidence}/release-performance.json" > /dev/null; then
            regressed=true
          fi
          echo "status=${status}" >> "${GITHUB_OUTPUT}"
          echo "regressed=${regressed}" >> "${GITHUB_OUTPUT}"
      - name: Upload comparison evidence
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: release-performance-${{ github.run_id }}-${{ github.run_attempt }}
          path: ${{ runner.temp }}/release-performance
          if-no-files-found: warn
          retention-days: 90
      - name: Fail on a regression or comparison error
        if: steps.compare.outputs.status != '0'
        env:
          STATUS: ${{ steps.compare.outputs.status }}
        run: |
          echo "The release performance comparison failed with exit status ${STATUS:-unknown}" >&2
          exit 1

  report:
    name: Report regression
    needs: compare
    if: ${{ always() && github.event_name == 'push' && needs.compare.outputs.regressed == 'true' }}
    runs-on: ubuntu-24.04
    # Overlapping main runs report one at a time so they update a single regression issue.
    concurrency:
      group: release-performance-report
      cancel-in-progress: false
    permissions:
      issues: write
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: release-performance-${{ github.run_id }}-${{ github.run_attempt }}
          path: evidence
      - name: Open or update the performance regression issue
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
          LABEL: performance-regression
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          {
            echo "Release performance regressed at ${GITHUB_SHA} on main."
            echo
            echo "Comparison run: ${RUN_URL}"
            echo
            cat evidence/release-performance.md
          } > report.md
          gh label create "${LABEL}" --color B60205 \
            --description "Main is slower or larger than the latest release" 2>/dev/null || true
          issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \
            --json number --jq '.[0].number // empty')"
          if [[ -n "${issue}" ]]; then
            gh issue comment "${issue}" --body-file report.md
            echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}"
          else
            gh issue create --title "Release performance regression on main" \
              --label "${LABEL}" --body-file report.md
          fi