diff --git a/.github/scripts/check_role.sh b/.github/scripts/check_role.sh new file mode 100755 index 0000000..2a8f98f --- /dev/null +++ b/.github/scripts/check_role.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# Gate: verify an actor holds the Maintain or Admin role on the repo under test. +# +# This is the single source of truth for "who may launch GPU tests". It runs in +# two places: +# 1. Baked into the runner image at /opt/gsw/check_role.sh, called as the +# first step of workflow/gpu-tests.yaml. This is the AUTHORITATIVE gate: it +# covers every entry point, including a direct workflow_dispatch from the +# Actions tab (which needs only *write* access, so it would otherwise +# bypass the /gpu-test comment check entirely). +# 2. Checked into the repository and used by the /gpu-test +# comment workflow, which runs on a GitHub-hosted runner and therefore +# cannot reach /opt/gsw. That copy is a fast-fail UX nicety only. +# +# Living in the image is what makes (1) trustworthy: the gate cannot be edited +# by a pull request, only by rebuilding and redeploying the runner image. +# +# The default GITHUB_TOKEN bot is allowed through: when gpu-tests.yaml is +# dispatched by the comment workflow, github.actor is github-actions[bot], and +# that path was already role-checked upstream. +# +# Usage: check_role.sh +# Env: GH_TOKEN token with repo read access +# GITHUB_REPOSITORY owner/repo to check the role against +# Exit: 0 authorized, 1 not authorized (reason on stderr). +set -euo pipefail + +ACTOR="${1:?usage: check_role.sh }" +REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" + +if [[ "$ACTOR" == "github-actions[bot]" ]]; then + echo "actor=$ACTOR is the workflow bot (already gated upstream) β€” authorized" + exit 0 +fi + +# role_name is the granular role: admin / maintain / write / triage / read. +# author_association cannot distinguish maintain from write, so it is unusable +# for this check. +ROLE="$(gh api "repos/${REPO}/collaborators/${ACTOR}/permission" --jq '.role_name')" + +if [[ "$ROLE" == "admin" || "$ROLE" == "maintain" ]]; then + echo "actor=$ACTOR role=$ROLE β€” authorized" + exit 0 +fi + +echo "actor=$ACTOR role=${ROLE:-none} β€” NOT authorized (requires maintain or admin)" >&2 +exit 1 diff --git a/.github/scripts/gpu_test_command.sh b/.github/scripts/gpu_test_command.sh new file mode 100755 index 0000000..290df0c --- /dev/null +++ b/.github/scripts/gpu_test_command.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# Handle a /gpu-test* PR comment: verify the commenter holds Maintain/Admin, work +# out which test scope was asked for, then dispatch gpu-tests.yaml against the PR's +# head commit. On rejection, react πŸ‘Ž and reply naming the requirement. +# +# THIS SCRIPT DOES NOT KNOW THE TEST FAMILIES. It derives the suite name from the +# command instead: `/gpu-test` -> full, `/gpu-test-` -> x. So the list lives in +# exactly one place, gpu-tests.yaml's `suite` input, and adding a family is a +# one-file change there rather than an edit here that is easy to forget -- the old +# shape had a hardcoded case, and forgetting it meant a family that existed in the +# workflow, the mapping and the docs was still told "not a command". +# +# `/gpu-test-` is NOT accepted. Two layers reject an unknown name: +# +# 1. This script reads the `options:` list out of the checked-out workflow (see +# WORKFLOW_FILE) and declines locally -- in seconds, on a hosted runner, with +# a reply listing the families READ FROM THAT LIST so it cannot go stale. +# 2. If that read fails -- someone reformats `options:` into a block list -- the +# dispatch goes ahead and GitHub's own `type: choice` validation rejects it +# with a 422, which is caught below. So a reformat costs the nice message, +# never the enforcement. +# +# Deliberately NOT fully dynamic. With no `options:` the dispatch would succeed and +# the failure would land as a red GPU Tests run, having consumed a runner slot and a +# queue wait, for a typo. There is no upside either: a name with no `options:` entry +# has no mapping arm to run. +# +# The suite NAME is dispatched, never a path list: gpu-tests.yaml owns the mapping. +# +# Deployed to granite-switch as .github/scripts/gpu_test_command.sh. +# It runs on a GitHub-hosted runner (no /opt/gsw), which is why it is checked in +# rather than being baked into the runner image. See +# gpu-test-command.yaml for the full rationale. +# +# This check is fast-fail UX. The authoritative gate is /opt/gsw/check_role.sh +# inside gpu-tests.yaml, which also covers direct workflow_dispatch. +# +# All GitHub-controlled values arrive as positional args from quoted env in the +# workflow β€” never interpolated into this script β€” so a crafted login cannot +# inject shell. +# +# Usage: gpu_test_command.sh +# Env: GH_TOKEN, GITHUB_REPOSITORY, DEFAULT_BRANCH, SCRIPT_DIR +# WORKFLOW_FILE optional path to the checked-out gpu-tests.yaml. Enables the +# local family check; without it layer 2 above still applies. +set -euo pipefail + +ACTOR="${1:?usage: gpu_test_command.sh }" +PR_NUMBER="${2:?missing pr number}" +COMMENT_ID="${3:?missing comment id}" +# May legitimately be empty or multi-line, so no :? guard. +BODY="${4:-}" + +REPO="${GITHUB_REPOSITORY:?}" +DEFAULT_BRANCH="${DEFAULT_BRANCH:?}" +SCRIPT_DIR="${SCRIPT_DIR:?}" +WORKFLOW_FILE="${WORKFLOW_FILE:-}" + +react() { + gh api -X POST "repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="$1" >/dev/null +} + +reply() { + gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="$1" >/dev/null +} + +# check_role.sh exits non-zero (and prints the role) when not authorized. +if ! ROLE_MSG="$("${SCRIPT_DIR}/check_role.sh" "$ACTOR" 2>&1)"; then + react '-1' + reply "@${ACTOR} the GPU test commands require the **Maintain** or **Admin** role. Not launching." + echo "$ROLE_MSG" >&2 + exit 1 +fi + +# The families, read from the ONE place they are defined: the `options:` line of +# the `suite` input in the checked-out gpu-tests.yaml. Space-separated, or empty if +# the file is absent or the line is not in flow style -- in which case the dispatch +# below is left to GitHub to validate. +# +# One awk, no pipe: splitting on [ and ] puts the list body in $2, and `exit` stops +# at the first match. (A pipe into head would risk SIGPIPE under `pipefail`.) +FAMILIES="" +if [[ -n "$WORKFLOW_FILE" && -r "$WORKFLOW_FILE" ]]; then + FAMILIES="$(awk -F'[][]' ' + /^[[:space:]]*options:[[:space:]]*\[/ { gsub(/[ ,]+/, " ", $2); print $2; exit } + ' "$WORKFLOW_FILE" 2>/dev/null || true)" +fi + +# Render the families back as the commands a human types: `full` is the bare +# /gpu-test, everything else is suffixed. Used only in the decline message. +usage_list() { + local f out="" + for f in $FAMILIES; do + if [[ "$f" == "full" ]]; then out="${out}\`/gpu-test\` "; else out="${out}\`/gpu-test-${f}\` "; fi + done + printf '%s' "$out" +} + +# Exit 0 throughout: a mistyped command is user error, not a broken workflow, and a +# red X on the launcher would send someone hunting a bug that isn't there. +decline() { + local msg="@${ACTOR} $1" + # Built in steps rather than as one ${FAMILIES:+...} expansion: $'\n' inside that + # is honoured by bash but not by every shell, and a message that silently prints + # a literal $'\n\n' is not worth the saved line. + if [[ -n "$FAMILIES" ]]; then + msg="$msg"$'\n\n'"Available: $(usage_list)" + fi + react 'confused' + reply "$msg" + echo "declined: $2" >&2 + exit 0 +} + +# Which scope? First whitespace-delimited token of the FIRST line, so +# "/gpu-test-dev please" works and a command followed by prose or a second +# paragraph still parses. \r is stripped because GitHub sends CRLF line endings. +CMD="$(printf '%s' "$BODY" | head -n1 | tr -d '\r' | awk '{print $1}')" + +# Derive rather than look up. Note `/gpu-testing` does NOT match /gpu-test-* (the +# next character is `i`, not `-`), so the workflow's startsWith prefilter letting it +# through does not make it a command. +case "$CMD" in + /gpu-test) SUITE="full" ;; + /gpu-test-*) SUITE="${CMD#/gpu-test-}" ;; + *) decline "\`${CMD}\` is not a GPU test command." "not a command: '$CMD'" ;; +esac + +# The derived name comes from an attacker-controlled comment and ends up in an API +# request, so it is constrained to a shape a family name could plausibly have before +# it is used for anything. Also stops an empty `/gpu-test-` from being dispatched. +if [[ ! "$SUITE" =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]]; then + decline "\`${CMD}\` is not a GPU test command." "malformed family name: '$SUITE'" +fi + +# Layer 1: local check against the list, when it could be read. +if [[ -n "$FAMILIES" ]]; then + case " $FAMILIES " in + *" $SUITE "*) : ;; + *) decline "there is no \`${SUITE}\` test family." "unknown family: '$SUITE'" ;; + esac +fi + +SHA="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha')" + +# Layer 2: GitHub's own `type: choice` validation. Only reachable when FAMILIES +# could not be read, since layer 1 would have caught it otherwise. +# +# The reaction is posted AFTER a successful dispatch, not before: a πŸš€ followed by +# "no such family" reads as though something launched and then broke. +if ! gh workflow run gpu-tests.yaml \ + --ref "$DEFAULT_BRANCH" \ + -f sha="$SHA" \ + -f pr_number="$PR_NUMBER" \ + -f suite="$SUITE" 2>/tmp/gh_dispatch_err; then + echo "dispatch failed:" >&2 + cat /tmp/gh_dispatch_err >&2 + decline "could not launch \`${SUITE}\` β€” it is probably not a valid test family." \ + "dispatch rejected for suite='$SUITE'" +fi + +react 'rocket' + +echo "Dispatched gpu-tests.yaml (suite=${SUITE}) for PR #${PR_NUMBER} at ${SHA} (by ${ACTOR})" diff --git a/.github/workflows/gpu-test-command.yaml b/.github/workflows/gpu-test-command.yaml new file mode 100644 index 0000000..39974b1 --- /dev/null +++ b/.github/workflows/gpu-test-command.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Slash-command launcher for /gpu-test, /gpu-test-short and /gpu-test-dev. +# +# The `if:` below is a cheap prefix prefilter, NOT the command parser: it lets any +# /gpu-test* comment start this job, and gpu_test_command.sh then matches the +# command exactly and declines anything else. Keeping the list in one place means +# adding a command does not need an edit here. +# +# Deployed to granite-switch as .github/workflows/gpu-test-command.yaml, together +# with .github/scripts/check_role.sh and .github/scripts/gpu_test_command.sh. +# +# WHY THOSE TWO SCRIPTS ARE CHECKED IN RATHER THAN BAKED INTO THE RUNNER IMAGE: +# this job runs on a GitHub-HOSTED runner, which cannot read the image's scripts, +# so they must come from the repository. Both contain only GitHub API calls. +# +# This is a convenience entry point, NOT the security boundary: gpu-tests.yaml +# re-checks the role via /opt/gsw/check_role.sh, which a pull request cannot edit. +# +# NOTE: issue_comment workflows only fire when the file is on the DEFAULT branch. +# It will not react to comments until merged to main. + +name: GPU Test Command + +on: + issue_comment: + types: [created] + +permissions: + actions: write # dispatch gpu-tests.yaml + pull-requests: write # react to the comment and post feedback + contents: read + +jobs: + dispatch: + name: Dispatch GPU tests + runs-on: ubuntu-latest + # Only for `/gpu-test` comments on a pull request (not plain issues). + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/gpu-test') + steps: + - name: Checkout scripts (trusted default branch) + uses: actions/checkout@v4 + + - name: Handle /gpu-test command + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + SCRIPT_DIR: ${{ github.workspace }}/.github/scripts + # The test families live in this file's `suite` input and nowhere else. + # The handler reads them out of it to validate the command and to build + # its decline message, so adding a family is a one-file change there. + WORKFLOW_FILE: ${{ github.workspace }}/.github/workflows/gpu-tests.yaml + # GitHub-controlled values pass through env, never interpolated into the + # script body β€” prevents shell injection via a crafted login. + ACTOR: ${{ github.event.comment.user.login }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + # The body decides WHICH scope runs (/gpu-test, -short or -dev). Entirely + # attacker-controlled text, so it follows the same rule as the login: env + # var, then a quoted positional arg, never an Actions expression inside the + # run block. + BODY: ${{ github.event.comment.body }} + run: .github/scripts/gpu_test_command.sh "$ACTOR" "$PR_NUMBER" "$COMMENT_ID" "$BODY" diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index ad1c975..3676cf8 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -1,23 +1,397 @@ # SPDX-License-Identifier: Apache-2.0 +# +# GPU Tests β€” launched via /gpu-test, /gpu-test-short or /gpu-test-dev, and runs on +# the self-hosted GPU runner. The three commands differ only in the `suite` input +# below; each reports under its own status context so a quick check cannot mask a +# full-suite failure. +# +# COPY THIS FILE INTO: granite-switch/.github/workflows/gpu-tests.yaml +# +# HOW IT SATISFIES THE THREE REQUIREMENTS: +# 1. Runs on our GPUs β€” `runs-on: [self-hosted, gpu]` targets the self-hosted +# runner, which launches the actual tests on the GPU cluster. +# 2. Maintainer-only β€” `workflow_dispatch` only (no pull_request trigger, so +# fork PRs can never auto-run on the hardware) AND the `check_role.sh` gate +# below, which rejects anyone without the Maintain or Admin role. The gate +# lives in the image, not the repo, so a PR cannot edit it β€” and it covers +# EVERY entry point, including a direct dispatch from the Actions tab +# (which needs only *write* access and would otherwise bypass the +# /gpu-test comment check). This replaces the old `environment: gpu` +# required-reviewer approval, so runs no longer pause for a click. +# 3. Result on the PR β€” posts a commit STATUS against the tested SHA, a +# sticky PR COMMENT, and uploads the full pytest log as an ARTIFACT. +# +# The workflow itself holds no GPU and no cluster credentials, and contains no +# inline logic: every step invokes a script baked into the runner image at +# /opt/gsw on the runner image. That keeps infrastructure details out of this +# repo and keeps this file readable. name: GPU Tests on: - workflow_dispatch: # admin-only trigger (requires write access) + workflow_dispatch: + inputs: + sha: + description: "Full commit SHA to test (exact checkout on the GPU cluster)." + required: true + type: string + pr_number: + description: "PR number to comment on (optional; derived from the SHA when omitted)." + required: false + type: string + # A CLOSED VOCABULARY, not a path string. This is dispatchable by anyone with + # *write* access from the Actions tab, and raw paths would reach both a + # `sed s#__TEST_PATHS__#…#g` (a literal # breaks the delimiter) and the pod's + # `pytest __TEST_PATHS__` command line. Three names resolved in one trusted + # place β€” the step below β€” removes that surface entirely. + # + # The mapping lives here rather than in the runner image so changing it needs + # no rebuild. It is equally trusted either way: gpu_test_command.sh dispatches + # with --ref , so this file always comes from main and a pull + # request cannot alter its own test scope. + # THE list of test families. gpu_test_command.sh derives the suite name from + # the comment (/gpu-test- -> x) and reads this line to validate it, so a new + # family is added HERE and nowhere else -- this entry plus its arm in the + # Resolve test scope step below. + suite: + description: "Which tests to run: full (all five suites), short (vllm + integration), dev (one fast GPU file), audio (the audio/ASR marker), multi (all five on 2 GPUs)." + required: false + default: full + type: choice + options: [full, short, dev, audio, multi] + +# Per-SHA-and-suite concurrency: re-dispatching the same commit with the same scope +# cancels its stale run. Different PRs (different SHAs) may run in parallel; excess +# cluster pods simply sit Pending until GPUs free up. +# +# The suite is in the group on purpose. Without it, a quick /gpu-test-dev on a commit +# would CANCEL the multi-hour /gpu-test already running against it β€” losing most of a +# day's work to a two-minute smoke check. +# +# NOTE: this is workflow-level, so the matrix legs below share one group and do +# NOT cancel each other. +concurrency: + group: gpu-tests-${{ inputs.sha || github.sha }}-${{ inputs.suite || 'full' }} + cancel-in-progress: true + +permissions: + statuses: write # post the commit status check + pull-requests: write # post/update the sticky PR comment + contents: read jobs: gpu-tests: - name: GPU Tests + # The suite is in the job name because it cannot be read from env.LEG here -- + # this is evaluated before any step runs. + name: GPU Tests (${{ matrix.label }} Β· ${{ inputs.suite || 'full' }}) + # granite-switch supports two mutually-exclusive vLLM lines and both must be + # tested. They cannot share a venv (pyproject declares vllm19/vllm20 as + # conflicting groups), so each gets its own cluster pod. Only the dev* groups + # used because the bare vllm19/vllm20 groups omit pytest. + strategy: + fail-fast: false # a vllm19 failure must not hide the vllm20 result + matrix: + include: + - label: vllm19 + dep_group: dev + - label: vllm20 + dep_group: dev-vllm20 runs-on: [self-hosted, gpu] + + # MUST be set explicitly. GitHub's default is 360 minutes, which is exactly the + # in-pod deadline, so GitHub would cancel the job at the same moment the pod's + # own timeout fires -- and the script's watchdog at 390 min could never run at + # all. A five-suite run is expected to exceed 360 min outright, so on the + # default this job gets cancelled mid-run regardless of whether the tests pass. + # + # Three related numbers, from innermost out. Raise one and check the others: + # in-pod `timeout` 21600s = 360 min (submit_and_poll.sh DEADLINE) + # script watchdog 23400s = 390 min (DEADLINE + 1800) + # this job cap 420 min (watchdog + 30 min of slack) + # + # Ordered so the innermost limit is normally what ends a wedged run: the pod + # kills its own tests, else the script reaps the job, and GitHub cancelling is + # the last resort -- which is the only one of the three that leaves an orphaned + # AppWrapper if the SIGTERM trap does not land. + timeout-minutes: 420 + + # Resolve the commit under test once, so every step below agrees on it. + env: + TARGET_SHA: ${{ inputs.sha || github.sha }} + # Clone from whichever repo dispatched this run, so a fork tests its own + # commits without editing anything. + TARGET_REPO: github.com/${{ github.repository }}.git steps: - - uses: actions/checkout@v4 + # No checkout: submit_and_poll.sh and the helper scripts are baked into the + # runner image at /opt/gsw. Avoids a cross-instance checkout and keeps + # cluster configuration out of this repo. + + # Turn the suite NAME into the three things that depend on it, once, so no + # later step has to know the mapping. Pure string work β€” no cluster access, no + # credentials β€” so it is safe to run ahead of the role gate. + # + # TEST_PATHS pytest targets, or empty for the full suite + # GPU_COUNT GPUs per pod, or empty to take submit_and_poll.sh's default + # JOB_SUFFIX keeps the cluster job name distinct (see LEG note below) + # LEG the reporting identity: status context, sticky comment, + # artifact, dashboard label + # + # LEG matters more than it looks. Everything published keys off it, and if a + # narrowed run reused the plain label, a passing /gpu-test-dev would OVERWRITE + # a failing /gpu-test β€” same status context, same sticky-comment marker β€” and + # the PR would look healthy while the full suite was still broken. A full run + # keeps the bare label, so existing statuses keep their identity. + - name: Resolve test scope + env: + SUITE: ${{ inputs.suite || 'full' }} + LABEL: ${{ matrix.label }} + run: | + set -euo pipefail + # Default for every scope: empty, meaning no --gpus flag, meaning + # submit_and_poll.sh's own default of 1. Set once here rather than in each + # arm so a new arm cannot silently inherit another's GPU count. + PATHS="" + SUFFIX="" + GPU_COUNT="" + case "$SUITE" in + # No --tests at all: submit_and_poll.sh's committed default stays the + # single definition of "everything", so adding a suite there needs no + # change here. + full) + PATHS="" + SUFFIX="" + LEG="$LABEL" + ;; + # The two suites that need CUDA. + short) + PATHS="tests/vllm/ tests/integration/" + SUFFIX="short" + LEG="$LABEL-short" + ;; + # One fast file that still exercises the GPU β€” ~30s of pytest, a couple + # of minutes end to end once pod startup and uv sync are counted. A + # CPU-only pick would pass with a broken CUDA venv. + dev) + PATHS="tests/vllm/test_single_switch.py" + SUFFIX="dev" + LEG="$LABEL-dev" + ;; + # The audio/ASR family, selected by MARKER rather than by path: the + # `audio` mark spans 8 files across unit, composer, vllm and integration, + # so a path list would be long and would drift silently every time a + # marked test is added. A marker cannot drift. + # + # "and not deep" is not optional. pytest's -m is single-valued and the + # command line REPLACES pyproject's addopts `-m "not deep"` rather than + # adding to it, so a bare `-m audio` would quietly start running the + # expensive deep audio tests that every other scope excludes. + # + # The quotes survive sed -> YAML -> helm -> the pod's shell; the + # neighbouring `python -c "..."` lines in the job template rely on the + # same thing. The one forbidden character is #, the sed delimiter. + audio) + PATHS='-m "audio and not deep" tests/' + SUFFIX="audio" + LEG="$LABEL-audio" + ;; + # Everything `full` runs, but on TWO GPUs. Not a different test + # selection -- a different SHAPE of pod. + # + # This is the only scope that unlocks tests rather than narrowing them: + # tests/vllm/test_tp_integration.py and + # tests/vllm/test_pipeline_parallelism_generation.py both carry + # `skipif(device_count() < 2)`, so on the 1-GPU pods every other scope + # uses they have NEVER run -- they are part of the "19 skipped" in a + # normal result line. + # + # Deliberately kept separate from `full` rather than raising `full` to 2: + # doubling every run's GPU demand to unlock a handful of tests is a bad + # trade, and `full` is what runs most often. + multi) + PATHS="" + SUFFIX="multi" + LEG="$LABEL-multi" + GPU_COUNT="2" + ;; + # Fail rather than defaulting to full: silently running the wrong scope + # is worse than not running, and `type: choice` already rejects unknown + # values, so reaching this means the options list above gained an entry + # without an arm here. + *) + echo "::error::unknown suite '$SUITE' β€” no mapping arm for it" + exit 1 + ;; + esac + { + echo "TEST_PATHS=$PATHS" + echo "JOB_SUFFIX=$SUFFIX" + echo "LEG=$LEG" + echo "GPU_COUNT=$GPU_COUNT" + } >> "$GITHUB_ENV" + echo "suite=$SUITE leg=$LEG gpus=${GPU_COUNT:-1} tests=${PATHS:-}" + + # The gate. First step that touches anything outside this runner. + - name: Verify launcher is Maintain/Admin + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + ACTOR: ${{ github.actor }} + run: /opt/gsw/check_role.sh "$ACTOR" + + - name: Post pending status + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + /opt/gsw/post_status.sh "$TARGET_SHA" pending \ + "GPU tests running on the cluster…" "${{ env.LEG }}" - - uses: astral-sh/setup-uv@v5 + # Register the run as in-progress on the internal dashboard, so a multi-hour + # run is visible while it is still going rather than only after it ends. + # Best-effort by construction: the script exits 0 even when the dashboard is + # unreachable or unconfigured, and continue-on-error means a problem here can + # never fail the GPU run. The dashboard address is NOT in this file β€” it comes + # from GSW_DASHBOARD_URL on the runner, so nothing internal is disclosed here. + - name: Register run start + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GSW_RUN_KEY: ${{ github.run_id }}-${{ env.LEG }} + GSW_REPO: ${{ github.repository }} + GSW_SHA: ${{ env.TARGET_SHA }} + GSW_PR: ${{ inputs.pr_number }} + GSW_LABEL: ${{ env.LEG }} + GSW_DEP_GROUP: ${{ matrix.dep_group }} + GSW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Nothing this script prints reaches the public console. It is telemetry + # for whoever runs the dashboard, of no use to a public reader, and on a + # curl failure it quotes the internal service host it could not reach. + # + # A file rather than /dev/null: the script always exits 0, so this output + # is the only evidence archiving failed, and the runner streams step logs + # to GitHub rather than to the container console, so `oc logs` would not + # show it. Read it with: + # oc exec deploy/gsw-gpu-listener -- tail -40 /tmp/archive_run.log + run: /opt/gsw/archive_run.sh start >>/tmp/archive_run.log 2>&1 + + - name: Run GPU tests on the cluster + id: run + # --raw-log sends the verbose run to a FILE. Nothing verbose reaches this + # console, because stdout here is the GitHub Actions job log: PUBLIC on a + # public repo, and the surface people actually open. All it receives is the + # heartbeat the script synthesizes from a fixed vocabulary of phase words + # and elapsed minutes. + # + # This replaces piping the run through a regex redactor. That worked most + # of the time, which is the problem -- a denylist publishes anything it has + # no rule for, and it was demonstrably leaking a quoted namespace and a + # .svc hostname. Nothing arbitrary is printed here now, so there is nothing + # to get wrong. + # + # The two consumers of the detail are unaffected: the next step builds the + # public report from the file (allowlisted -- header fields by name, the + # pytest block by range), and the archive step ships the full file to the + # internal dashboard. + env: + # submit_and_poll.sh reports the pod name to the dashboard as soon as it + # submits, and needs the same run key the start step used to address the + # row. Without this the report silently no-ops and the pod column stays + # empty until finish. + GSW_RUN_KEY: ${{ github.run_id }}-${{ env.LEG }} + run: | + # Both flags are omitted entirely when empty, which is the `full` case: + # no --tests means the script's own default applies, so "everything" has + # exactly one definition. Read from env with ${VAR:+...} rather than through + # an Actions expression -- the value must reach the script as ONE argument, + # and nothing GitHub-controlled belongs in this shell. + # + # Do NOT write an empty Actions expression anywhere in a run: block, not + # even in a comment. Block-scalar content is scanned for expressions (YAML + # comments are stripped before that, block scalars are not), so one cost a + # dispatch with "HTTP 422 ... An expression was expected". + # + # --job-suffix keeps the cluster job name unique per scope. The name is + # gsw-gpu--, and the script DELETES a pre-existing job of the + # same name before submitting β€” so without the suffix, launching a dev run + # would reap the full run already testing that commit. + /opt/gsw/submit_and_poll.sh \ + --sha "$TARGET_SHA" \ + --repo "$TARGET_REPO" \ + --dep-group "${{ matrix.dep_group }}" \ + ${TEST_PATHS:+--tests "$TEST_PATHS"} \ + ${GPU_COUNT:+--gpus "$GPU_COUNT"} \ + ${JOB_SUFFIX:+--job-suffix "$JOB_SUFFIX"} \ + --raw-log gpu-tests-raw.log + # Cluster credentials come from the runner's own environment, never from + # GitHub secrets β€” this workflow neither holds nor sees them. + + # The raw log is for cluster debugging and stays on the runner. Everything + # published β€” this console, the artifact and the PR comment β€” comes from the + # cleaned report, which keeps the pytest output verbatim and redacts + # infrastructure detail. if: always() so a failed run still produces a + # readable report. + # + # tee, so the report is also printed here: with the verbose run no longer + # streaming, this is the only thing that puts test results in the job log, + # and "which test failed?" should not require downloading an artifact. + - name: Build public test report + if: always() + run: | + set -o pipefail + /opt/gsw/clean_log.sh gpu-tests-raw.log "${{ env.LEG }}" | tee gpu-tests.log + + # Send the COMPLETE raw log to the internal dashboard β€” the only place it is + # kept. The run itself already shipped it in batches, so this is the + # authoritative copy that replaces those: it closes any chunk that failed to + # send, and covers the case where nothing shipped at all. Same best-effort + # contract as the start step. + - name: Archive raw log + if: always() + continue-on-error: true + env: + GSW_RUN_KEY: ${{ github.run_id }}-${{ env.LEG }} + # Redirected for the same reason as the start step: nothing here is for a + # public reader, and the failure path names the internal dashboard host. + run: | + { + if [[ "${{ steps.run.outcome }}" == "success" ]]; then + /opt/gsw/archive_run.sh finish passed gpu-tests-raw.log + elif [[ "${{ steps.run.outcome }}" == "cancelled" ]]; then + /opt/gsw/archive_run.sh finish cancelled gpu-tests-raw.log + else + /opt/gsw/archive_run.sh finish failed gpu-tests-raw.log + fi + } >>/tmp/archive_run.log 2>&1 + + - name: Upload test log + if: always() + uses: actions/upload-artifact@v4 with: - enable-cache: true + name: gpu-tests-log-${{ env.LEG }}-${{ env.TARGET_SHA }} + path: gpu-tests.log + if-no-files-found: warn - - run: uv sync --frozen --group dev --extra hf --extra vllm --extra compose + - name: Post final commit status + if: always() + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + if [[ "${{ steps.run.outcome }}" == "success" ]]; then + /opt/gsw/post_status.sh "$TARGET_SHA" success "GPU tests passed" "${{ env.LEG }}" + else + /opt/gsw/post_status.sh "$TARGET_SHA" failure "GPU tests failed" "${{ env.LEG }}" + fi - - name: Run GPU tests + - name: Update sticky PR comment + # Runs on pass AND fail (that is the point β€” a failure comment carries the + # log excerpt). Whether it actually comments depends on resolving a PR, + # which the script handles. + if: always() + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ inputs.pr_number }} run: | - uv run pytest tests/vllm/ tests/integration/ -v -s --tb=short -x + /opt/gsw/update_sticky_comment.sh "$TARGET_SHA" "${{ steps.run.outcome }}" \ + gpu-tests.log "${{ env.LEG }}" "$PR_NUMBER"