From 8063e9d7a99c5a8d54cd028cf01d169667c14f3c Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 00:27:07 +0200 Subject: [PATCH 01/10] ci: run CodeSpeed benchmarks on Nx agents --- .github/workflows/client-nav-benchmarks.yml | 18 +- .github/workflows/pr.yml | 9 + .nx/workflows/dynamic-changesets.yaml | 7 + benchmarks/client-nav/package.json | 78 +++++++- benchmarks/memory/client/package.json | 66 ++++++- benchmarks/memory/server/package.json | 66 ++++++- benchmarks/ssr/package.json | 66 ++++++- scripts/nx/run-codspeed-benchmark.mjs | 205 ++++++++++++++++++++ 8 files changed, 486 insertions(+), 29 deletions(-) create mode 100644 scripts/nx/run-codspeed-benchmark.mjs diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index a56bb70334..4aad64d842 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -2,23 +2,7 @@ name: Benchmarks on: - push: - # Broad on purpose: CodSpeed compares each PR against the run on its - # merge-base commit. Skipping main pushes that don't touch packages/** - # (pnpm config, CI, root tooling) leaves those commits without a baseline - # run, and CodSpeed silently falls back to an older, possibly outlier run — - # which then flags phantom regressions on every PR until the next eligible - # push. Only paths that cannot affect benchmark results are excluded. - branches: - - 'main' - paths-ignore: - - 'docs/**' - - 'examples/**' - - 'e2e/**' - pull_request: - paths: - - 'packages/**' - - 'benchmarks/**' + # CI runs these benchmarks through the PR workflow's Nx Agents. workflow_dispatch: permissions: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 71026a0a92..a49407d66f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,6 +13,7 @@ env: permissions: contents: read + id-token: write jobs: test: @@ -36,6 +37,14 @@ jobs: main-branch-name: main - name: Run Checks run: pnpm run test:pr --parallel=3 + - name: Run CodeSpeed Benchmarks + env: + CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} + run: | + pnpm nx run-many \ + --projects=@benchmarks/client-nav,@benchmarks/ssr,@benchmarks/memory-client,@benchmarks/memory-server \ + --targets=test:codspeed:react,test:codspeed:solid,test:codspeed:vue \ + --outputStyle=stream - name: Stop Nx Agents if: ${{ always() }} run: npx nx-cloud stop-all-agents diff --git a/.nx/workflows/dynamic-changesets.yaml b/.nx/workflows/dynamic-changesets.yaml index 786111053d..478f19d5d2 100644 --- a/.nx/workflows/dynamic-changesets.yaml +++ b/.nx/workflows/dynamic-changesets.yaml @@ -2,3 +2,10 @@ distribute-on: small-changeset: 4 linux-medium-js medium-changeset: 8 linux-medium-js large-changeset: 14 linux-medium-js + +assignment-rules: + - targets: + - 'test:codspeed:*' + run-on: + - agent: linux-medium-js + parallelism: 1 diff --git a/benchmarks/client-nav/package.json b/benchmarks/client-nav/package.json index 701e068b49..096f8461ef 100644 --- a/benchmarks/client-nav/package.json +++ b/benchmarks/client-nav/package.json @@ -13,6 +13,9 @@ "test:perf:react": "NODE_ENV=production vitest bench --config ./vitest.react.config.ts", "test:perf:solid": "NODE_ENV=production vitest bench --config ./vitest.solid.config.ts", "test:perf:vue": "NODE_ENV=production vitest bench --config ./vitest.vue.config.ts", + "test:codspeed:react": "node ../../scripts/nx/run-codspeed-benchmark.mjs client-nav react", + "test:codspeed:solid": "node ../../scripts/nx/run-codspeed-benchmark.mjs client-nav solid", + "test:codspeed:vue": "node ../../scripts/nx/run-codspeed-benchmark.mjs client-nav vue", "test:types": "pnpm run test:types:react && pnpm run test:types:solid && pnpm run test:types:vue", "test:types:react": "tsc -p ./react/tsconfig.json --noEmit", "test:types:solid": "tsc -p ./solid/tsconfig.json --noEmit", @@ -45,7 +48,14 @@ "nx": { "targets": { "build:react": { - "cache": false, + "cache": true, + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/react/dist" + ], "dependsOn": [ { "projects": [ @@ -73,7 +83,14 @@ ] }, "build:solid": { - "cache": false, + "cache": true, + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/solid/dist" + ], "dependsOn": [ { "projects": [ @@ -101,7 +118,14 @@ ] }, "build:vue": { - "cache": false, + "cache": true, + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/vue/dist" + ], "dependsOn": [ { "projects": [ @@ -172,6 +196,54 @@ "build:vue" ] }, + "test:codspeed:react": { + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ] + }, + "test:codspeed:solid": { + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ] + }, + "test:codspeed:vue": { + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ] + }, "test:types": { "cache": false, "dependsOn": [ diff --git a/benchmarks/memory/client/package.json b/benchmarks/memory/client/package.json index 2624b9e4e5..8f8f5fe668 100644 --- a/benchmarks/memory/client/package.json +++ b/benchmarks/memory/client/package.json @@ -40,7 +40,7 @@ "targets": { "build:react": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -57,7 +57,7 @@ }, "build:solid": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -74,7 +74,7 @@ }, "build:vue": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -227,6 +227,66 @@ "cwd": "benchmarks/memory/client" } }, + "test:codspeed:react": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-client react" + } + }, + "test:codspeed:solid": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-client solid" + } + }, + "test:codspeed:vue": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-client vue" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index 68ba6b3c32..58f8587525 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -37,7 +37,7 @@ "targets": { "build:react": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -55,7 +55,7 @@ }, "build:solid": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -73,7 +73,7 @@ }, "build:vue": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -233,6 +233,66 @@ "cwd": "benchmarks/memory/server" } }, + "test:codspeed:react": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-server react" + } + }, + "test:codspeed:solid": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-server solid" + } + }, + "test:codspeed:vue": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-server vue" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ diff --git a/benchmarks/ssr/package.json b/benchmarks/ssr/package.json index 40cd97df5e..6ebb14affd 100644 --- a/benchmarks/ssr/package.json +++ b/benchmarks/ssr/package.json @@ -28,7 +28,7 @@ "targets": { "build:react": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -54,7 +54,7 @@ }, "build:solid": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -80,7 +80,7 @@ }, "build:vue": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -150,6 +150,66 @@ "cwd": "benchmarks/ssr" } }, + "test:codspeed:react": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs ssr react" + } + }, + "test:codspeed:solid": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs ssr solid" + } + }, + "test:codspeed:vue": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs ssr vue" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ diff --git a/scripts/nx/run-codspeed-benchmark.mjs b/scripts/nx/run-codspeed-benchmark.mjs new file mode 100644 index 0000000000..97973c5d9d --- /dev/null +++ b/scripts/nx/run-codspeed-benchmark.mjs @@ -0,0 +1,205 @@ +import { createHash } from 'node:crypto' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { spawn } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const CODSPEED_VERSION = '4.17.0' +const CODSPEED_INSTALLER_SHA256 = + 'b5151c275a2b8055593cc20e7d3d0258059e07c8b633a7bb2519df9306f9fd67' + +const rootDir = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) + +const benchmarkConfigs = { + 'client-nav': { + mode: 'simulation', + cwd: 'benchmarks/client-nav', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, + ssr: { + mode: 'simulation', + cwd: 'benchmarks/ssr', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, + 'memory-client': { + mode: 'memory', + cwd: 'benchmarks/memory/client', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, + 'memory-server': { + mode: 'memory', + cwd: 'benchmarks/memory/server', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, +} + +const [benchmark, framework] = process.argv.slice(2) +const benchmarkConfig = benchmarkConfigs[benchmark] +const vitestConfig = benchmarkConfig?.frameworks[framework] + +if (!benchmarkConfig || !vitestConfig) { + console.error( + `Usage: node scripts/nx/run-codspeed-benchmark.mjs <${Object.keys( + benchmarkConfigs, + ).join('|')}> `, + ) + process.exit(1) +} + +const codspeed = await findCodSpeedBinary() +const codspeedArgs = [ + 'run', + `--mode=${process.env.CODSPEED_MODE ?? benchmarkConfig.mode}`, +] + +if (process.env.CODSPEED_TOKEN) { + codspeedArgs.push('--token', process.env.CODSPEED_TOKEN) +} + +codspeedArgs.push( + '--', + 'bash', + '-lc', + `NODE_ENV=production vitest bench --config ./${vitestConfig}`, +) + +await execFile(codspeed, codspeedArgs, { + cwd: join(rootDir, benchmarkConfig.cwd), + env: { + ...process.env, + WITH_INSTRUMENTATION: '1', + }, +}) + +async function findCodSpeedBinary() { + const path = await which('codspeed') + + if (path) { + return path + } + + await installCodSpeed() + + const installedPath = await which('codspeed', { + PATH: buildPathWithCodSpeedInstallDirs(), + }) + + if (installedPath) { + process.env.PATH = buildPathWithCodSpeedInstallDirs() + return installedPath + } + + throw new Error('Unable to find codspeed after installing the runner') +} + +async function installCodSpeed() { + const dir = await mkdtemp(join(tmpdir(), 'codspeed-runner-')) + const installer = join(dir, 'install.sh') + + try { + const body = await fetch( + `https://codspeed.io/v${CODSPEED_VERSION}/install.sh`, + ) + + if (!body.ok) { + throw new Error( + `Failed to download CodSpeed installer: HTTP ${body.status}`, + ) + } + + const buffer = Buffer.from(await body.arrayBuffer()) + const actualHash = createHash('sha256').update(buffer).digest('hex') + + if (actualHash !== CODSPEED_INSTALLER_SHA256) { + throw new Error( + `CodSpeed installer hash mismatch. Expected ${CODSPEED_INSTALLER_SHA256}, got ${actualHash}.`, + ) + } + + await writeFile(installer, buffer, { mode: 0o755 }) + await execFile('bash', [installer, '--quiet']) + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +async function which(binary, env = process.env) { + try { + const { stdout } = await execFile('bash', ['-lc', `command -v ${binary}`], { + env, + capture: true, + }) + + return stdout.trim() || undefined + } catch { + return undefined + } +} + +function buildPathWithCodSpeedInstallDirs() { + const home = process.env.HOME + + if (!home) { + return process.env.PATH ?? '' + } + + return [ + join(home, '.codspeed', 'bin'), + join(home, '.cargo', 'bin'), + join(home, '.local', 'bin'), + process.env.PATH ?? '', + ].join(':') +} + +async function execFile(file, args, options = {}) { + const child = spawn(file, args, { + cwd: options.cwd, + env: options.env, + stdio: options.capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', + }) + + let stdout = '' + let stderr = '' + + if (options.capture) { + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + stdout += chunk + }) + child.stderr.on('data', (chunk) => { + stderr += chunk + }) + } + + const code = await new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', resolve) + }) + + if (code !== 0) { + const message = options.capture + ? stderr || stdout || `${file} exited with code ${code}` + : `${file} exited with code ${code}` + throw new Error(message) + } + + return { stdout, stderr } +} From f13f89dc6ec64350decddeb307130fdf6a838a6b Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 07:32:34 +0200 Subject: [PATCH 02/10] fix(ci): guard CodeSpeed agent auth --- .github/workflows/pr.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a49407d66f..ce60126010 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -13,7 +13,6 @@ env: permissions: contents: read - id-token: write jobs: test: @@ -41,6 +40,11 @@ jobs: env: CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} run: | + if [ -z "$CODSPEED_TOKEN" ]; then + echo "::notice::Skipping CodeSpeed benchmarks because CODSPEED_TOKEN is not configured for Nx Agent execution." + exit 0 + fi + pnpm nx run-many \ --projects=@benchmarks/client-nav,@benchmarks/ssr,@benchmarks/memory-client,@benchmarks/memory-server \ --targets=test:codspeed:react,test:codspeed:solid,test:codspeed:vue \ From 4404b7b2c585f37a161bd96f93e089f22ceafd7a Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 08:38:56 +0200 Subject: [PATCH 03/10] ci: retry CodeSpeed benchmarks with token From 086bc3c5c62b9bb54dfc208eaed39c40bbedb72c Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 08:51:43 +0200 Subject: [PATCH 04/10] fix(ci): forward CodeSpeed token to Nx agents --- .github/workflows/pr.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ce60126010..c7ced65ec6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -19,6 +19,7 @@ jobs: name: Test runs-on: ubuntu-latest env: + CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} TSR_TMP_DIR: ./tmp steps: - name: Checkout @@ -27,7 +28,7 @@ jobs: fetch-depth: 0 persist-credentials: false - name: Start Nx Agents - run: npx nx-cloud start-ci-run --distribute-on=".nx/workflows/dynamic-changesets.yaml" + run: npx nx-cloud start-ci-run --distribute-on=".nx/workflows/dynamic-changesets.yaml" --with-env-vars="CODSPEED_TOKEN" - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - name: Get base and head commits for `nx affected` @@ -37,8 +38,6 @@ jobs: - name: Run Checks run: pnpm run test:pr --parallel=3 - name: Run CodeSpeed Benchmarks - env: - CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} run: | if [ -z "$CODSPEED_TOKEN" ]; then echo "::notice::Skipping CodeSpeed benchmarks because CODSPEED_TOKEN is not configured for Nx Agent execution." From f89616ef9ee54308555adb1d02d913ca00f6f104 Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 09:03:42 +0200 Subject: [PATCH 05/10] fix(ci): use CodeSpeed OIDC token on agents --- .github/workflows/pr.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c7ced65ec6..eab40de219 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -18,8 +18,10 @@ jobs: test: name: Test runs-on: ubuntu-latest + permissions: + contents: read + id-token: write env: - CODSPEED_TOKEN: ${{ secrets.CODSPEED_TOKEN }} TSR_TMP_DIR: ./tmp steps: - name: Checkout @@ -28,7 +30,14 @@ jobs: fetch-depth: 0 persist-credentials: false - name: Start Nx Agents - run: npx nx-cloud start-ci-run --distribute-on=".nx/workflows/dynamic-changesets.yaml" --with-env-vars="CODSPEED_TOKEN" + run: | + oidc_response=$(curl -fsSL -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=codspeed.io") + CODSPEED_TOKEN=$(node -e 'const fs = require("node:fs"); const { value } = JSON.parse(fs.readFileSync(0, "utf8")); if (!value) { process.exit(1) } process.stdout.write(value)' <<< "$oidc_response") + echo "::add-mask::$CODSPEED_TOKEN" + echo "CODSPEED_TOKEN=$CODSPEED_TOKEN" >> "$GITHUB_ENV" + export CODSPEED_TOKEN + + npx nx-cloud start-ci-run --distribute-on=".nx/workflows/dynamic-changesets.yaml" --with-env-vars="CODSPEED_TOKEN" - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - name: Get base and head commits for `nx affected` From 64f490ab5bcd7ae3d2cbbf7d61b29ed5f5a4f8aa Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 09:20:37 +0200 Subject: [PATCH 06/10] fix(ci): forward CodeSpeed OIDC request env to agents --- .github/workflows/pr.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index eab40de219..5d06899ba4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -31,13 +31,9 @@ jobs: persist-credentials: false - name: Start Nx Agents run: | - oidc_response=$(curl -fsSL -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=codspeed.io") - CODSPEED_TOKEN=$(node -e 'const fs = require("node:fs"); const { value } = JSON.parse(fs.readFileSync(0, "utf8")); if (!value) { process.exit(1) } process.stdout.write(value)' <<< "$oidc_response") - echo "::add-mask::$CODSPEED_TOKEN" - echo "CODSPEED_TOKEN=$CODSPEED_TOKEN" >> "$GITHUB_ENV" - export CODSPEED_TOKEN - - npx nx-cloud start-ci-run --distribute-on=".nx/workflows/dynamic-changesets.yaml" --with-env-vars="CODSPEED_TOKEN" + npx nx-cloud start-ci-run \ + --distribute-on=".nx/workflows/dynamic-changesets.yaml" \ + --with-env-vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CODSPEED_TOKEN" - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - name: Get base and head commits for `nx affected` @@ -48,8 +44,8 @@ jobs: run: pnpm run test:pr --parallel=3 - name: Run CodeSpeed Benchmarks run: | - if [ -z "$CODSPEED_TOKEN" ]; then - echo "::notice::Skipping CodeSpeed benchmarks because CODSPEED_TOKEN is not configured for Nx Agent execution." + if [ -z "$CODSPEED_TOKEN" ] && { [ -z "$ACTIONS_ID_TOKEN_REQUEST_TOKEN" ] || [ -z "$ACTIONS_ID_TOKEN_REQUEST_URL" ]; }; then + echo "::notice::Skipping CodeSpeed benchmarks because neither CODSPEED_TOKEN nor GitHub OIDC auth is configured for Nx Agent execution." exit 0 fi From 4b683a2ab35d4acc70ded9fdeab3991e4fa92d80 Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 09:36:57 +0200 Subject: [PATCH 07/10] fix(ci): forward GitHub context to CodeSpeed agents --- .github/workflows/pr.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5d06899ba4..a54a6a091d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -31,9 +31,11 @@ jobs: persist-credentials: false - name: Start Nx Agents run: | + codspeed_env_vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CI,GITHUB_ACTIONS,GITHUB_API_URL,GITHUB_BASE_REF,GITHUB_HEAD_REF,GITHUB_REF,GITHUB_REF_NAME,GITHUB_REF_TYPE,GITHUB_REPOSITORY,GITHUB_REPOSITORY_ID,GITHUB_RUN_ATTEMPT,GITHUB_RUN_ID,GITHUB_RUN_NUMBER,GITHUB_SERVER_URL,GITHUB_SHA,GITHUB_WORKFLOW,GITHUB_WORKFLOW_REF,GITHUB_WORKFLOW_SHA" + npx nx-cloud start-ci-run \ --distribute-on=".nx/workflows/dynamic-changesets.yaml" \ - --with-env-vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CODSPEED_TOKEN" + --with-env-vars="$codspeed_env_vars" - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - name: Get base and head commits for `nx affected` From bc7742cdd7225dbda5afb30f69587dff0f4fba06 Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 09:45:56 +0200 Subject: [PATCH 08/10] fix(ci): provide GitHub event payload for CodeSpeed agents --- .github/workflows/pr.yml | 2 +- scripts/nx/run-codspeed-benchmark.mjs | 55 +++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a54a6a091d..5f9e696faf 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -31,7 +31,7 @@ jobs: persist-credentials: false - name: Start Nx Agents run: | - codspeed_env_vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CI,GITHUB_ACTIONS,GITHUB_API_URL,GITHUB_BASE_REF,GITHUB_HEAD_REF,GITHUB_REF,GITHUB_REF_NAME,GITHUB_REF_TYPE,GITHUB_REPOSITORY,GITHUB_REPOSITORY_ID,GITHUB_RUN_ATTEMPT,GITHUB_RUN_ID,GITHUB_RUN_NUMBER,GITHUB_SERVER_URL,GITHUB_SHA,GITHUB_WORKFLOW,GITHUB_WORKFLOW_REF,GITHUB_WORKFLOW_SHA" + codspeed_env_vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CI,GITHUB_ACTIONS,GITHUB_API_URL,GITHUB_BASE_REF,GITHUB_EVENT_NAME,GITHUB_HEAD_REF,GITHUB_REF,GITHUB_REF_NAME,GITHUB_REF_TYPE,GITHUB_REPOSITORY,GITHUB_REPOSITORY_ID,GITHUB_RUN_ATTEMPT,GITHUB_RUN_ID,GITHUB_RUN_NUMBER,GITHUB_SERVER_URL,GITHUB_SHA,GITHUB_WORKFLOW,GITHUB_WORKFLOW_REF,GITHUB_WORKFLOW_SHA" npx nx-cloud start-ci-run \ --distribute-on=".nx/workflows/dynamic-changesets.yaml" \ diff --git a/scripts/nx/run-codspeed-benchmark.mjs b/scripts/nx/run-codspeed-benchmark.mjs index 97973c5d9d..f3b009485b 100644 --- a/scripts/nx/run-codspeed-benchmark.mjs +++ b/scripts/nx/run-codspeed-benchmark.mjs @@ -63,6 +63,8 @@ if (!benchmarkConfig || !vitestConfig) { process.exit(1) } +await ensureGithubEventPath() + const codspeed = await findCodSpeedBinary() const codspeedArgs = [ 'run', @@ -88,6 +90,59 @@ await execFile(codspeed, codspeedArgs, { }, }) +async function ensureGithubEventPath() { + if (process.env.GITHUB_EVENT_PATH || process.env.GITHUB_ACTIONS !== 'true') { + return + } + + const eventDir = await mkdtemp(join(tmpdir(), 'codspeed-github-event-')) + const eventPath = join(eventDir, 'event.json') + const repository = process.env.GITHUB_REPOSITORY ?? '' + const pullRequestNumber = getPullRequestNumber() + const event = { + repository: { + full_name: repository, + }, + ...(pullRequestNumber ? { number: pullRequestNumber } : {}), + pull_request: { + ...(pullRequestNumber ? { number: pullRequestNumber } : {}), + head: { + ref: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? '', + sha: process.env.GITHUB_SHA ?? '', + repo: { + full_name: repository, + }, + }, + base: { + ref: process.env.GITHUB_BASE_REF ?? '', + repo: { + full_name: repository, + }, + }, + }, + } + + await writeFile(eventPath, JSON.stringify(event), 'utf8') + process.env.GITHUB_EVENT_PATH = eventPath +} + +function getPullRequestNumber() { + const match = process.env.GITHUB_REF?.match(/^refs\/pull\/(\d+)\//) + + if (match) { + return Number.parseInt(match[1], 10) + } + + const refName = process.env.GITHUB_REF_NAME ?? '' + const refNameMatch = refName.match(/^(\d+)\//) + + if (refNameMatch) { + return Number.parseInt(refNameMatch[1], 10) + } + + return undefined +} + async function findCodSpeedBinary() { const path = await which('codspeed') From 776770ebb4b63b96613eb61ae82a5c320e4e7a30 Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 09:56:07 +0200 Subject: [PATCH 09/10] fix(ci): complete CodeSpeed GitHub metadata on agents --- .github/workflows/pr.yml | 2 +- scripts/nx/run-codspeed-benchmark.mjs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5f9e696faf..a087a77540 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -31,7 +31,7 @@ jobs: persist-credentials: false - name: Start Nx Agents run: | - codspeed_env_vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CI,GITHUB_ACTIONS,GITHUB_API_URL,GITHUB_BASE_REF,GITHUB_EVENT_NAME,GITHUB_HEAD_REF,GITHUB_REF,GITHUB_REF_NAME,GITHUB_REF_TYPE,GITHUB_REPOSITORY,GITHUB_REPOSITORY_ID,GITHUB_RUN_ATTEMPT,GITHUB_RUN_ID,GITHUB_RUN_NUMBER,GITHUB_SERVER_URL,GITHUB_SHA,GITHUB_WORKFLOW,GITHUB_WORKFLOW_REF,GITHUB_WORKFLOW_SHA" + codspeed_env_vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CI,GITHUB_ACTIONS,GITHUB_ACTOR,GITHUB_ACTOR_ID,GITHUB_API_URL,GITHUB_BASE_REF,GITHUB_EVENT_NAME,GITHUB_HEAD_REF,GITHUB_JOB,GITHUB_REF,GITHUB_REF_NAME,GITHUB_REF_TYPE,GITHUB_REPOSITORY,GITHUB_REPOSITORY_ID,GITHUB_RUN_ATTEMPT,GITHUB_RUN_ID,GITHUB_RUN_NUMBER,GITHUB_SERVER_URL,GITHUB_SHA,GITHUB_WORKFLOW,GITHUB_WORKFLOW_REF,GITHUB_WORKFLOW_SHA,GITHUB_WORKSPACE" npx nx-cloud start-ci-run \ --distribute-on=".nx/workflows/dynamic-changesets.yaml" \ diff --git a/scripts/nx/run-codspeed-benchmark.mjs b/scripts/nx/run-codspeed-benchmark.mjs index f3b009485b..35ab0ac229 100644 --- a/scripts/nx/run-codspeed-benchmark.mjs +++ b/scripts/nx/run-codspeed-benchmark.mjs @@ -98,10 +98,13 @@ async function ensureGithubEventPath() { const eventDir = await mkdtemp(join(tmpdir(), 'codspeed-github-event-')) const eventPath = join(eventDir, 'event.json') const repository = process.env.GITHUB_REPOSITORY ?? '' + const repositoryId = Number.parseInt(process.env.GITHUB_REPOSITORY_ID ?? '0', 10) const pullRequestNumber = getPullRequestNumber() const event = { repository: { full_name: repository, + id: repositoryId, + private: true, }, ...(pullRequestNumber ? { number: pullRequestNumber } : {}), pull_request: { @@ -111,12 +114,14 @@ async function ensureGithubEventPath() { sha: process.env.GITHUB_SHA ?? '', repo: { full_name: repository, + id: repositoryId, }, }, base: { ref: process.env.GITHUB_BASE_REF ?? '', repo: { full_name: repository, + id: repositoryId, }, }, }, From aa05c842d28cbd094fc866d311406ec36973f584 Mon Sep 17 00:00:00 2001 From: Nicolas Beaussart Date: Thu, 9 Jul 2026 10:06:04 +0200 Subject: [PATCH 10/10] fix(ci): run supported CodeSpeed modes on agents --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a087a77540..d4c6c9c0b1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -52,7 +52,7 @@ jobs: fi pnpm nx run-many \ - --projects=@benchmarks/client-nav,@benchmarks/ssr,@benchmarks/memory-client,@benchmarks/memory-server \ + --projects=@benchmarks/client-nav,@benchmarks/ssr \ --targets=test:codspeed:react,test:codspeed:solid,test:codspeed:vue \ --outputStyle=stream - name: Stop Nx Agents