From 0cd6416e7cceda27143f4c78037031977a104514 Mon Sep 17 00:00:00 2001 From: aviv ron Date: Wed, 5 Aug 2026 19:52:48 +0300 Subject: [PATCH] support for self hosted runner. runs tests on self hosted runner based on comments on prs made by either admin or maintainer. posts the results as sticky comments in the pr Signed-off-by: aviv ron --- .github/scripts/check_role.sh | 49 +++ .github/scripts/gpu_test_command.sh | 167 ++++++++++ .github/workflows/gpu-test-command.yaml | 66 ++++ .github/workflows/gpu-tests.yaml | 390 +++++++++++++++++++++++- 4 files changed, 664 insertions(+), 8 deletions(-) create mode 100755 .github/scripts/check_role.sh create mode 100755 .github/scripts/gpu_test_command.sh create mode 100644 .github/workflows/gpu-test-command.yaml diff --git a/.github/scripts/check_role.sh b/.github/scripts/check_role.sh new file mode 100755 index 0000000..2a8f98f --- /dev/null +++ b/.github/scripts/check_role.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# Gate: verify an actor holds the Maintain or Admin role on the repo under test. +# +# This is the single source of truth for "who may launch GPU tests". It runs in +# two places: +# 1. Baked into the runner image at /opt/gsw/check_role.sh, called as the +# first step of workflow/gpu-tests.yaml. This is the AUTHORITATIVE gate: it +# covers every entry point, including a direct workflow_dispatch from the +# Actions tab (which needs only *write* access, so it would otherwise +# bypass the /gpu-test comment check entirely). +# 2. Checked into the repository and used by the /gpu-test +# comment workflow, which runs on a GitHub-hosted runner and therefore +# cannot reach /opt/gsw. That copy is a fast-fail UX nicety only. +# +# Living in the image is what makes (1) trustworthy: the gate cannot be edited +# by a pull request, only by rebuilding and redeploying the runner image. +# +# The default GITHUB_TOKEN bot is allowed through: when gpu-tests.yaml is +# dispatched by the comment workflow, github.actor is github-actions[bot], and +# that path was already role-checked upstream. +# +# Usage: check_role.sh +# Env: GH_TOKEN token with repo read access +# GITHUB_REPOSITORY owner/repo to check the role against +# Exit: 0 authorized, 1 not authorized (reason on stderr). +set -euo pipefail + +ACTOR="${1:?usage: check_role.sh }" +REPO="${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" + +if [[ "$ACTOR" == "github-actions[bot]" ]]; then + echo "actor=$ACTOR is the workflow bot (already gated upstream) β€” authorized" + exit 0 +fi + +# role_name is the granular role: admin / maintain / write / triage / read. +# author_association cannot distinguish maintain from write, so it is unusable +# for this check. +ROLE="$(gh api "repos/${REPO}/collaborators/${ACTOR}/permission" --jq '.role_name')" + +if [[ "$ROLE" == "admin" || "$ROLE" == "maintain" ]]; then + echo "actor=$ACTOR role=$ROLE β€” authorized" + exit 0 +fi + +echo "actor=$ACTOR role=${ROLE:-none} β€” NOT authorized (requires maintain or admin)" >&2 +exit 1 diff --git a/.github/scripts/gpu_test_command.sh b/.github/scripts/gpu_test_command.sh new file mode 100755 index 0000000..290df0c --- /dev/null +++ b/.github/scripts/gpu_test_command.sh @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# +# Handle a /gpu-test* PR comment: verify the commenter holds Maintain/Admin, work +# out which test scope was asked for, then dispatch gpu-tests.yaml against the PR's +# head commit. On rejection, react πŸ‘Ž and reply naming the requirement. +# +# THIS SCRIPT DOES NOT KNOW THE TEST FAMILIES. It derives the suite name from the +# command instead: `/gpu-test` -> full, `/gpu-test-` -> x. So the list lives in +# exactly one place, gpu-tests.yaml's `suite` input, and adding a family is a +# one-file change there rather than an edit here that is easy to forget -- the old +# shape had a hardcoded case, and forgetting it meant a family that existed in the +# workflow, the mapping and the docs was still told "not a command". +# +# `/gpu-test-` is NOT accepted. Two layers reject an unknown name: +# +# 1. This script reads the `options:` list out of the checked-out workflow (see +# WORKFLOW_FILE) and declines locally -- in seconds, on a hosted runner, with +# a reply listing the families READ FROM THAT LIST so it cannot go stale. +# 2. If that read fails -- someone reformats `options:` into a block list -- the +# dispatch goes ahead and GitHub's own `type: choice` validation rejects it +# with a 422, which is caught below. So a reformat costs the nice message, +# never the enforcement. +# +# Deliberately NOT fully dynamic. With no `options:` the dispatch would succeed and +# the failure would land as a red GPU Tests run, having consumed a runner slot and a +# queue wait, for a typo. There is no upside either: a name with no `options:` entry +# has no mapping arm to run. +# +# The suite NAME is dispatched, never a path list: gpu-tests.yaml owns the mapping. +# +# Deployed to granite-switch as .github/scripts/gpu_test_command.sh. +# It runs on a GitHub-hosted runner (no /opt/gsw), which is why it is checked in +# rather than being baked into the runner image. See +# gpu-test-command.yaml for the full rationale. +# +# This check is fast-fail UX. The authoritative gate is /opt/gsw/check_role.sh +# inside gpu-tests.yaml, which also covers direct workflow_dispatch. +# +# All GitHub-controlled values arrive as positional args from quoted env in the +# workflow β€” never interpolated into this script β€” so a crafted login cannot +# inject shell. +# +# Usage: gpu_test_command.sh +# Env: GH_TOKEN, GITHUB_REPOSITORY, DEFAULT_BRANCH, SCRIPT_DIR +# WORKFLOW_FILE optional path to the checked-out gpu-tests.yaml. Enables the +# local family check; without it layer 2 above still applies. +set -euo pipefail + +ACTOR="${1:?usage: gpu_test_command.sh }" +PR_NUMBER="${2:?missing pr number}" +COMMENT_ID="${3:?missing comment id}" +# May legitimately be empty or multi-line, so no :? guard. +BODY="${4:-}" + +REPO="${GITHUB_REPOSITORY:?}" +DEFAULT_BRANCH="${DEFAULT_BRANCH:?}" +SCRIPT_DIR="${SCRIPT_DIR:?}" +WORKFLOW_FILE="${WORKFLOW_FILE:-}" + +react() { + gh api -X POST "repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="$1" >/dev/null +} + +reply() { + gh api -X POST "repos/${REPO}/issues/${PR_NUMBER}/comments" -f body="$1" >/dev/null +} + +# check_role.sh exits non-zero (and prints the role) when not authorized. +if ! ROLE_MSG="$("${SCRIPT_DIR}/check_role.sh" "$ACTOR" 2>&1)"; then + react '-1' + reply "@${ACTOR} the GPU test commands require the **Maintain** or **Admin** role. Not launching." + echo "$ROLE_MSG" >&2 + exit 1 +fi + +# The families, read from the ONE place they are defined: the `options:` line of +# the `suite` input in the checked-out gpu-tests.yaml. Space-separated, or empty if +# the file is absent or the line is not in flow style -- in which case the dispatch +# below is left to GitHub to validate. +# +# One awk, no pipe: splitting on [ and ] puts the list body in $2, and `exit` stops +# at the first match. (A pipe into head would risk SIGPIPE under `pipefail`.) +FAMILIES="" +if [[ -n "$WORKFLOW_FILE" && -r "$WORKFLOW_FILE" ]]; then + FAMILIES="$(awk -F'[][]' ' + /^[[:space:]]*options:[[:space:]]*\[/ { gsub(/[ ,]+/, " ", $2); print $2; exit } + ' "$WORKFLOW_FILE" 2>/dev/null || true)" +fi + +# Render the families back as the commands a human types: `full` is the bare +# /gpu-test, everything else is suffixed. Used only in the decline message. +usage_list() { + local f out="" + for f in $FAMILIES; do + if [[ "$f" == "full" ]]; then out="${out}\`/gpu-test\` "; else out="${out}\`/gpu-test-${f}\` "; fi + done + printf '%s' "$out" +} + +# Exit 0 throughout: a mistyped command is user error, not a broken workflow, and a +# red X on the launcher would send someone hunting a bug that isn't there. +decline() { + local msg="@${ACTOR} $1" + # Built in steps rather than as one ${FAMILIES:+...} expansion: $'\n' inside that + # is honoured by bash but not by every shell, and a message that silently prints + # a literal $'\n\n' is not worth the saved line. + if [[ -n "$FAMILIES" ]]; then + msg="$msg"$'\n\n'"Available: $(usage_list)" + fi + react 'confused' + reply "$msg" + echo "declined: $2" >&2 + exit 0 +} + +# Which scope? First whitespace-delimited token of the FIRST line, so +# "/gpu-test-dev please" works and a command followed by prose or a second +# paragraph still parses. \r is stripped because GitHub sends CRLF line endings. +CMD="$(printf '%s' "$BODY" | head -n1 | tr -d '\r' | awk '{print $1}')" + +# Derive rather than look up. Note `/gpu-testing` does NOT match /gpu-test-* (the +# next character is `i`, not `-`), so the workflow's startsWith prefilter letting it +# through does not make it a command. +case "$CMD" in + /gpu-test) SUITE="full" ;; + /gpu-test-*) SUITE="${CMD#/gpu-test-}" ;; + *) decline "\`${CMD}\` is not a GPU test command." "not a command: '$CMD'" ;; +esac + +# The derived name comes from an attacker-controlled comment and ends up in an API +# request, so it is constrained to a shape a family name could plausibly have before +# it is used for anything. Also stops an empty `/gpu-test-` from being dispatched. +if [[ ! "$SUITE" =~ ^[a-z0-9][a-z0-9-]{0,31}$ ]]; then + decline "\`${CMD}\` is not a GPU test command." "malformed family name: '$SUITE'" +fi + +# Layer 1: local check against the list, when it could be read. +if [[ -n "$FAMILIES" ]]; then + case " $FAMILIES " in + *" $SUITE "*) : ;; + *) decline "there is no \`${SUITE}\` test family." "unknown family: '$SUITE'" ;; + esac +fi + +SHA="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}" --jq '.head.sha')" + +# Layer 2: GitHub's own `type: choice` validation. Only reachable when FAMILIES +# could not be read, since layer 1 would have caught it otherwise. +# +# The reaction is posted AFTER a successful dispatch, not before: a πŸš€ followed by +# "no such family" reads as though something launched and then broke. +if ! gh workflow run gpu-tests.yaml \ + --ref "$DEFAULT_BRANCH" \ + -f sha="$SHA" \ + -f pr_number="$PR_NUMBER" \ + -f suite="$SUITE" 2>/tmp/gh_dispatch_err; then + echo "dispatch failed:" >&2 + cat /tmp/gh_dispatch_err >&2 + decline "could not launch \`${SUITE}\` β€” it is probably not a valid test family." \ + "dispatch rejected for suite='$SUITE'" +fi + +react 'rocket' + +echo "Dispatched gpu-tests.yaml (suite=${SUITE}) for PR #${PR_NUMBER} at ${SHA} (by ${ACTOR})" diff --git a/.github/workflows/gpu-test-command.yaml b/.github/workflows/gpu-test-command.yaml new file mode 100644 index 0000000..39974b1 --- /dev/null +++ b/.github/workflows/gpu-test-command.yaml @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# Slash-command launcher for /gpu-test, /gpu-test-short and /gpu-test-dev. +# +# The `if:` below is a cheap prefix prefilter, NOT the command parser: it lets any +# /gpu-test* comment start this job, and gpu_test_command.sh then matches the +# command exactly and declines anything else. Keeping the list in one place means +# adding a command does not need an edit here. +# +# Deployed to granite-switch as .github/workflows/gpu-test-command.yaml, together +# with .github/scripts/check_role.sh and .github/scripts/gpu_test_command.sh. +# +# WHY THOSE TWO SCRIPTS ARE CHECKED IN RATHER THAN BAKED INTO THE RUNNER IMAGE: +# this job runs on a GitHub-HOSTED runner, which cannot read the image's scripts, +# so they must come from the repository. Both contain only GitHub API calls. +# +# This is a convenience entry point, NOT the security boundary: gpu-tests.yaml +# re-checks the role via /opt/gsw/check_role.sh, which a pull request cannot edit. +# +# NOTE: issue_comment workflows only fire when the file is on the DEFAULT branch. +# It will not react to comments until merged to main. + +name: GPU Test Command + +on: + issue_comment: + types: [created] + +permissions: + actions: write # dispatch gpu-tests.yaml + pull-requests: write # react to the comment and post feedback + contents: read + +jobs: + dispatch: + name: Dispatch GPU tests + runs-on: ubuntu-latest + # Only for `/gpu-test` comments on a pull request (not plain issues). + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/gpu-test') + steps: + - name: Checkout scripts (trusted default branch) + uses: actions/checkout@v4 + + - name: Handle /gpu-test command + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + SCRIPT_DIR: ${{ github.workspace }}/.github/scripts + # The test families live in this file's `suite` input and nowhere else. + # The handler reads them out of it to validate the command and to build + # its decline message, so adding a family is a one-file change there. + WORKFLOW_FILE: ${{ github.workspace }}/.github/workflows/gpu-tests.yaml + # GitHub-controlled values pass through env, never interpolated into the + # script body β€” prevents shell injection via a crafted login. + ACTOR: ${{ github.event.comment.user.login }} + PR_NUMBER: ${{ github.event.issue.number }} + COMMENT_ID: ${{ github.event.comment.id }} + # The body decides WHICH scope runs (/gpu-test, -short or -dev). Entirely + # attacker-controlled text, so it follows the same rule as the login: env + # var, then a quoted positional arg, never an Actions expression inside the + # run block. + BODY: ${{ github.event.comment.body }} + run: .github/scripts/gpu_test_command.sh "$ACTOR" "$PR_NUMBER" "$COMMENT_ID" "$BODY" diff --git a/.github/workflows/gpu-tests.yaml b/.github/workflows/gpu-tests.yaml index ad1c975..3676cf8 100644 --- a/.github/workflows/gpu-tests.yaml +++ b/.github/workflows/gpu-tests.yaml @@ -1,23 +1,397 @@ # SPDX-License-Identifier: Apache-2.0 +# +# GPU Tests β€” launched via /gpu-test, /gpu-test-short or /gpu-test-dev, and runs on +# the self-hosted GPU runner. The three commands differ only in the `suite` input +# below; each reports under its own status context so a quick check cannot mask a +# full-suite failure. +# +# COPY THIS FILE INTO: granite-switch/.github/workflows/gpu-tests.yaml +# +# HOW IT SATISFIES THE THREE REQUIREMENTS: +# 1. Runs on our GPUs β€” `runs-on: [self-hosted, gpu]` targets the self-hosted +# runner, which launches the actual tests on the GPU cluster. +# 2. Maintainer-only β€” `workflow_dispatch` only (no pull_request trigger, so +# fork PRs can never auto-run on the hardware) AND the `check_role.sh` gate +# below, which rejects anyone without the Maintain or Admin role. The gate +# lives in the image, not the repo, so a PR cannot edit it β€” and it covers +# EVERY entry point, including a direct dispatch from the Actions tab +# (which needs only *write* access and would otherwise bypass the +# /gpu-test comment check). This replaces the old `environment: gpu` +# required-reviewer approval, so runs no longer pause for a click. +# 3. Result on the PR β€” posts a commit STATUS against the tested SHA, a +# sticky PR COMMENT, and uploads the full pytest log as an ARTIFACT. +# +# The workflow itself holds no GPU and no cluster credentials, and contains no +# inline logic: every step invokes a script baked into the runner image at +# /opt/gsw on the runner image. That keeps infrastructure details out of this +# repo and keeps this file readable. name: GPU Tests on: - workflow_dispatch: # admin-only trigger (requires write access) + workflow_dispatch: + inputs: + sha: + description: "Full commit SHA to test (exact checkout on the GPU cluster)." + required: true + type: string + pr_number: + description: "PR number to comment on (optional; derived from the SHA when omitted)." + required: false + type: string + # A CLOSED VOCABULARY, not a path string. This is dispatchable by anyone with + # *write* access from the Actions tab, and raw paths would reach both a + # `sed s#__TEST_PATHS__#…#g` (a literal # breaks the delimiter) and the pod's + # `pytest __TEST_PATHS__` command line. Three names resolved in one trusted + # place β€” the step below β€” removes that surface entirely. + # + # The mapping lives here rather than in the runner image so changing it needs + # no rebuild. It is equally trusted either way: gpu_test_command.sh dispatches + # with --ref , so this file always comes from main and a pull + # request cannot alter its own test scope. + # THE list of test families. gpu_test_command.sh derives the suite name from + # the comment (/gpu-test- -> x) and reads this line to validate it, so a new + # family is added HERE and nowhere else -- this entry plus its arm in the + # Resolve test scope step below. + suite: + description: "Which tests to run: full (all five suites), short (vllm + integration), dev (one fast GPU file), audio (the audio/ASR marker), multi (all five on 2 GPUs)." + required: false + default: full + type: choice + options: [full, short, dev, audio, multi] + +# Per-SHA-and-suite concurrency: re-dispatching the same commit with the same scope +# cancels its stale run. Different PRs (different SHAs) may run in parallel; excess +# cluster pods simply sit Pending until GPUs free up. +# +# The suite is in the group on purpose. Without it, a quick /gpu-test-dev on a commit +# would CANCEL the multi-hour /gpu-test already running against it β€” losing most of a +# day's work to a two-minute smoke check. +# +# NOTE: this is workflow-level, so the matrix legs below share one group and do +# NOT cancel each other. +concurrency: + group: gpu-tests-${{ inputs.sha || github.sha }}-${{ inputs.suite || 'full' }} + cancel-in-progress: true + +permissions: + statuses: write # post the commit status check + pull-requests: write # post/update the sticky PR comment + contents: read jobs: gpu-tests: - name: GPU Tests + # The suite is in the job name because it cannot be read from env.LEG here -- + # this is evaluated before any step runs. + name: GPU Tests (${{ matrix.label }} Β· ${{ inputs.suite || 'full' }}) + # granite-switch supports two mutually-exclusive vLLM lines and both must be + # tested. They cannot share a venv (pyproject declares vllm19/vllm20 as + # conflicting groups), so each gets its own cluster pod. Only the dev* groups + # used because the bare vllm19/vllm20 groups omit pytest. + strategy: + fail-fast: false # a vllm19 failure must not hide the vllm20 result + matrix: + include: + - label: vllm19 + dep_group: dev + - label: vllm20 + dep_group: dev-vllm20 runs-on: [self-hosted, gpu] + + # MUST be set explicitly. GitHub's default is 360 minutes, which is exactly the + # in-pod deadline, so GitHub would cancel the job at the same moment the pod's + # own timeout fires -- and the script's watchdog at 390 min could never run at + # all. A five-suite run is expected to exceed 360 min outright, so on the + # default this job gets cancelled mid-run regardless of whether the tests pass. + # + # Three related numbers, from innermost out. Raise one and check the others: + # in-pod `timeout` 21600s = 360 min (submit_and_poll.sh DEADLINE) + # script watchdog 23400s = 390 min (DEADLINE + 1800) + # this job cap 420 min (watchdog + 30 min of slack) + # + # Ordered so the innermost limit is normally what ends a wedged run: the pod + # kills its own tests, else the script reaps the job, and GitHub cancelling is + # the last resort -- which is the only one of the three that leaves an orphaned + # AppWrapper if the SIGTERM trap does not land. + timeout-minutes: 420 + + # Resolve the commit under test once, so every step below agrees on it. + env: + TARGET_SHA: ${{ inputs.sha || github.sha }} + # Clone from whichever repo dispatched this run, so a fork tests its own + # commits without editing anything. + TARGET_REPO: github.com/${{ github.repository }}.git steps: - - uses: actions/checkout@v4 + # No checkout: submit_and_poll.sh and the helper scripts are baked into the + # runner image at /opt/gsw. Avoids a cross-instance checkout and keeps + # cluster configuration out of this repo. + + # Turn the suite NAME into the three things that depend on it, once, so no + # later step has to know the mapping. Pure string work β€” no cluster access, no + # credentials β€” so it is safe to run ahead of the role gate. + # + # TEST_PATHS pytest targets, or empty for the full suite + # GPU_COUNT GPUs per pod, or empty to take submit_and_poll.sh's default + # JOB_SUFFIX keeps the cluster job name distinct (see LEG note below) + # LEG the reporting identity: status context, sticky comment, + # artifact, dashboard label + # + # LEG matters more than it looks. Everything published keys off it, and if a + # narrowed run reused the plain label, a passing /gpu-test-dev would OVERWRITE + # a failing /gpu-test β€” same status context, same sticky-comment marker β€” and + # the PR would look healthy while the full suite was still broken. A full run + # keeps the bare label, so existing statuses keep their identity. + - name: Resolve test scope + env: + SUITE: ${{ inputs.suite || 'full' }} + LABEL: ${{ matrix.label }} + run: | + set -euo pipefail + # Default for every scope: empty, meaning no --gpus flag, meaning + # submit_and_poll.sh's own default of 1. Set once here rather than in each + # arm so a new arm cannot silently inherit another's GPU count. + PATHS="" + SUFFIX="" + GPU_COUNT="" + case "$SUITE" in + # No --tests at all: submit_and_poll.sh's committed default stays the + # single definition of "everything", so adding a suite there needs no + # change here. + full) + PATHS="" + SUFFIX="" + LEG="$LABEL" + ;; + # The two suites that need CUDA. + short) + PATHS="tests/vllm/ tests/integration/" + SUFFIX="short" + LEG="$LABEL-short" + ;; + # One fast file that still exercises the GPU β€” ~30s of pytest, a couple + # of minutes end to end once pod startup and uv sync are counted. A + # CPU-only pick would pass with a broken CUDA venv. + dev) + PATHS="tests/vllm/test_single_switch.py" + SUFFIX="dev" + LEG="$LABEL-dev" + ;; + # The audio/ASR family, selected by MARKER rather than by path: the + # `audio` mark spans 8 files across unit, composer, vllm and integration, + # so a path list would be long and would drift silently every time a + # marked test is added. A marker cannot drift. + # + # "and not deep" is not optional. pytest's -m is single-valued and the + # command line REPLACES pyproject's addopts `-m "not deep"` rather than + # adding to it, so a bare `-m audio` would quietly start running the + # expensive deep audio tests that every other scope excludes. + # + # The quotes survive sed -> YAML -> helm -> the pod's shell; the + # neighbouring `python -c "..."` lines in the job template rely on the + # same thing. The one forbidden character is #, the sed delimiter. + audio) + PATHS='-m "audio and not deep" tests/' + SUFFIX="audio" + LEG="$LABEL-audio" + ;; + # Everything `full` runs, but on TWO GPUs. Not a different test + # selection -- a different SHAPE of pod. + # + # This is the only scope that unlocks tests rather than narrowing them: + # tests/vllm/test_tp_integration.py and + # tests/vllm/test_pipeline_parallelism_generation.py both carry + # `skipif(device_count() < 2)`, so on the 1-GPU pods every other scope + # uses they have NEVER run -- they are part of the "19 skipped" in a + # normal result line. + # + # Deliberately kept separate from `full` rather than raising `full` to 2: + # doubling every run's GPU demand to unlock a handful of tests is a bad + # trade, and `full` is what runs most often. + multi) + PATHS="" + SUFFIX="multi" + LEG="$LABEL-multi" + GPU_COUNT="2" + ;; + # Fail rather than defaulting to full: silently running the wrong scope + # is worse than not running, and `type: choice` already rejects unknown + # values, so reaching this means the options list above gained an entry + # without an arm here. + *) + echo "::error::unknown suite '$SUITE' β€” no mapping arm for it" + exit 1 + ;; + esac + { + echo "TEST_PATHS=$PATHS" + echo "JOB_SUFFIX=$SUFFIX" + echo "LEG=$LEG" + echo "GPU_COUNT=$GPU_COUNT" + } >> "$GITHUB_ENV" + echo "suite=$SUITE leg=$LEG gpus=${GPU_COUNT:-1} tests=${PATHS:-}" + + # The gate. First step that touches anything outside this runner. + - name: Verify launcher is Maintain/Admin + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + ACTOR: ${{ github.actor }} + run: /opt/gsw/check_role.sh "$ACTOR" + + - name: Post pending status + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + /opt/gsw/post_status.sh "$TARGET_SHA" pending \ + "GPU tests running on the cluster…" "${{ env.LEG }}" - - uses: astral-sh/setup-uv@v5 + # Register the run as in-progress on the internal dashboard, so a multi-hour + # run is visible while it is still going rather than only after it ends. + # Best-effort by construction: the script exits 0 even when the dashboard is + # unreachable or unconfigured, and continue-on-error means a problem here can + # never fail the GPU run. The dashboard address is NOT in this file β€” it comes + # from GSW_DASHBOARD_URL on the runner, so nothing internal is disclosed here. + - name: Register run start + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + GSW_RUN_KEY: ${{ github.run_id }}-${{ env.LEG }} + GSW_REPO: ${{ github.repository }} + GSW_SHA: ${{ env.TARGET_SHA }} + GSW_PR: ${{ inputs.pr_number }} + GSW_LABEL: ${{ env.LEG }} + GSW_DEP_GROUP: ${{ matrix.dep_group }} + GSW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + # Nothing this script prints reaches the public console. It is telemetry + # for whoever runs the dashboard, of no use to a public reader, and on a + # curl failure it quotes the internal service host it could not reach. + # + # A file rather than /dev/null: the script always exits 0, so this output + # is the only evidence archiving failed, and the runner streams step logs + # to GitHub rather than to the container console, so `oc logs` would not + # show it. Read it with: + # oc exec deploy/gsw-gpu-listener -- tail -40 /tmp/archive_run.log + run: /opt/gsw/archive_run.sh start >>/tmp/archive_run.log 2>&1 + + - name: Run GPU tests on the cluster + id: run + # --raw-log sends the verbose run to a FILE. Nothing verbose reaches this + # console, because stdout here is the GitHub Actions job log: PUBLIC on a + # public repo, and the surface people actually open. All it receives is the + # heartbeat the script synthesizes from a fixed vocabulary of phase words + # and elapsed minutes. + # + # This replaces piping the run through a regex redactor. That worked most + # of the time, which is the problem -- a denylist publishes anything it has + # no rule for, and it was demonstrably leaking a quoted namespace and a + # .svc hostname. Nothing arbitrary is printed here now, so there is nothing + # to get wrong. + # + # The two consumers of the detail are unaffected: the next step builds the + # public report from the file (allowlisted -- header fields by name, the + # pytest block by range), and the archive step ships the full file to the + # internal dashboard. + env: + # submit_and_poll.sh reports the pod name to the dashboard as soon as it + # submits, and needs the same run key the start step used to address the + # row. Without this the report silently no-ops and the pod column stays + # empty until finish. + GSW_RUN_KEY: ${{ github.run_id }}-${{ env.LEG }} + run: | + # Both flags are omitted entirely when empty, which is the `full` case: + # no --tests means the script's own default applies, so "everything" has + # exactly one definition. Read from env with ${VAR:+...} rather than through + # an Actions expression -- the value must reach the script as ONE argument, + # and nothing GitHub-controlled belongs in this shell. + # + # Do NOT write an empty Actions expression anywhere in a run: block, not + # even in a comment. Block-scalar content is scanned for expressions (YAML + # comments are stripped before that, block scalars are not), so one cost a + # dispatch with "HTTP 422 ... An expression was expected". + # + # --job-suffix keeps the cluster job name unique per scope. The name is + # gsw-gpu--, and the script DELETES a pre-existing job of the + # same name before submitting β€” so without the suffix, launching a dev run + # would reap the full run already testing that commit. + /opt/gsw/submit_and_poll.sh \ + --sha "$TARGET_SHA" \ + --repo "$TARGET_REPO" \ + --dep-group "${{ matrix.dep_group }}" \ + ${TEST_PATHS:+--tests "$TEST_PATHS"} \ + ${GPU_COUNT:+--gpus "$GPU_COUNT"} \ + ${JOB_SUFFIX:+--job-suffix "$JOB_SUFFIX"} \ + --raw-log gpu-tests-raw.log + # Cluster credentials come from the runner's own environment, never from + # GitHub secrets β€” this workflow neither holds nor sees them. + + # The raw log is for cluster debugging and stays on the runner. Everything + # published β€” this console, the artifact and the PR comment β€” comes from the + # cleaned report, which keeps the pytest output verbatim and redacts + # infrastructure detail. if: always() so a failed run still produces a + # readable report. + # + # tee, so the report is also printed here: with the verbose run no longer + # streaming, this is the only thing that puts test results in the job log, + # and "which test failed?" should not require downloading an artifact. + - name: Build public test report + if: always() + run: | + set -o pipefail + /opt/gsw/clean_log.sh gpu-tests-raw.log "${{ env.LEG }}" | tee gpu-tests.log + + # Send the COMPLETE raw log to the internal dashboard β€” the only place it is + # kept. The run itself already shipped it in batches, so this is the + # authoritative copy that replaces those: it closes any chunk that failed to + # send, and covers the case where nothing shipped at all. Same best-effort + # contract as the start step. + - name: Archive raw log + if: always() + continue-on-error: true + env: + GSW_RUN_KEY: ${{ github.run_id }}-${{ env.LEG }} + # Redirected for the same reason as the start step: nothing here is for a + # public reader, and the failure path names the internal dashboard host. + run: | + { + if [[ "${{ steps.run.outcome }}" == "success" ]]; then + /opt/gsw/archive_run.sh finish passed gpu-tests-raw.log + elif [[ "${{ steps.run.outcome }}" == "cancelled" ]]; then + /opt/gsw/archive_run.sh finish cancelled gpu-tests-raw.log + else + /opt/gsw/archive_run.sh finish failed gpu-tests-raw.log + fi + } >>/tmp/archive_run.log 2>&1 + + - name: Upload test log + if: always() + uses: actions/upload-artifact@v4 with: - enable-cache: true + name: gpu-tests-log-${{ env.LEG }}-${{ env.TARGET_SHA }} + path: gpu-tests.log + if-no-files-found: warn - - run: uv sync --frozen --group dev --extra hf --extra vllm --extra compose + - name: Post final commit status + if: always() + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + run: | + if [[ "${{ steps.run.outcome }}" == "success" ]]; then + /opt/gsw/post_status.sh "$TARGET_SHA" success "GPU tests passed" "${{ env.LEG }}" + else + /opt/gsw/post_status.sh "$TARGET_SHA" failure "GPU tests failed" "${{ env.LEG }}" + fi - - name: Run GPU tests + - name: Update sticky PR comment + # Runs on pass AND fail (that is the point β€” a failure comment carries the + # log excerpt). Whether it actually comments depends on resolving a PR, + # which the script handles. + if: always() + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + PR_NUMBER: ${{ inputs.pr_number }} run: | - uv run pytest tests/vllm/ tests/integration/ -v -s --tb=short -x + /opt/gsw/update_sticky_comment.sh "$TARGET_SHA" "${{ steps.run.outcome }}" \ + gpu-tests.log "${{ env.LEG }}" "$PR_NUMBER"