diff --git a/.github/workflows/client-nav-benchmarks.yml b/.github/workflows/client-nav-benchmarks.yml index a56bb70334..4aad64d842 100644 --- a/.github/workflows/client-nav-benchmarks.yml +++ b/.github/workflows/client-nav-benchmarks.yml @@ -2,23 +2,7 @@ name: Benchmarks on: - push: - # Broad on purpose: CodSpeed compares each PR against the run on its - # merge-base commit. Skipping main pushes that don't touch packages/** - # (pnpm config, CI, root tooling) leaves those commits without a baseline - # run, and CodSpeed silently falls back to an older, possibly outlier run — - # which then flags phantom regressions on every PR until the next eligible - # push. Only paths that cannot affect benchmark results are excluded. - branches: - - 'main' - paths-ignore: - - 'docs/**' - - 'examples/**' - - 'e2e/**' - pull_request: - paths: - - 'packages/**' - - 'benchmarks/**' + # CI runs these benchmarks through the PR workflow's Nx Agents. workflow_dispatch: permissions: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 71026a0a92..d4c6c9c0b1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -18,6 +18,9 @@ jobs: test: name: Test runs-on: ubuntu-latest + permissions: + contents: read + id-token: write env: TSR_TMP_DIR: ./tmp steps: @@ -27,7 +30,12 @@ jobs: fetch-depth: 0 persist-credentials: false - name: Start Nx Agents - run: npx nx-cloud start-ci-run --distribute-on=".nx/workflows/dynamic-changesets.yaml" + run: | + codspeed_env_vars="ACTIONS_ID_TOKEN_REQUEST_TOKEN,ACTIONS_ID_TOKEN_REQUEST_URL,CI,GITHUB_ACTIONS,GITHUB_ACTOR,GITHUB_ACTOR_ID,GITHUB_API_URL,GITHUB_BASE_REF,GITHUB_EVENT_NAME,GITHUB_HEAD_REF,GITHUB_JOB,GITHUB_REF,GITHUB_REF_NAME,GITHUB_REF_TYPE,GITHUB_REPOSITORY,GITHUB_REPOSITORY_ID,GITHUB_RUN_ATTEMPT,GITHUB_RUN_ID,GITHUB_RUN_NUMBER,GITHUB_SERVER_URL,GITHUB_SHA,GITHUB_WORKFLOW,GITHUB_WORKFLOW_REF,GITHUB_WORKFLOW_SHA,GITHUB_WORKSPACE" + + npx nx-cloud start-ci-run \ + --distribute-on=".nx/workflows/dynamic-changesets.yaml" \ + --with-env-vars="$codspeed_env_vars" - name: Setup Tools uses: TanStack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 # main - name: Get base and head commits for `nx affected` @@ -36,6 +44,17 @@ jobs: main-branch-name: main - name: Run Checks run: pnpm run test:pr --parallel=3 + - name: Run CodeSpeed Benchmarks + run: | + if [ -z "$CODSPEED_TOKEN" ] && { [ -z "$ACTIONS_ID_TOKEN_REQUEST_TOKEN" ] || [ -z "$ACTIONS_ID_TOKEN_REQUEST_URL" ]; }; then + echo "::notice::Skipping CodeSpeed benchmarks because neither CODSPEED_TOKEN nor GitHub OIDC auth is configured for Nx Agent execution." + exit 0 + fi + + pnpm nx run-many \ + --projects=@benchmarks/client-nav,@benchmarks/ssr \ + --targets=test:codspeed:react,test:codspeed:solid,test:codspeed:vue \ + --outputStyle=stream - name: Stop Nx Agents if: ${{ always() }} run: npx nx-cloud stop-all-agents diff --git a/.nx/workflows/dynamic-changesets.yaml b/.nx/workflows/dynamic-changesets.yaml index 786111053d..478f19d5d2 100644 --- a/.nx/workflows/dynamic-changesets.yaml +++ b/.nx/workflows/dynamic-changesets.yaml @@ -2,3 +2,10 @@ distribute-on: small-changeset: 4 linux-medium-js medium-changeset: 8 linux-medium-js large-changeset: 14 linux-medium-js + +assignment-rules: + - targets: + - 'test:codspeed:*' + run-on: + - agent: linux-medium-js + parallelism: 1 diff --git a/benchmarks/client-nav/package.json b/benchmarks/client-nav/package.json index 701e068b49..096f8461ef 100644 --- a/benchmarks/client-nav/package.json +++ b/benchmarks/client-nav/package.json @@ -13,6 +13,9 @@ "test:perf:react": "NODE_ENV=production vitest bench --config ./vitest.react.config.ts", "test:perf:solid": "NODE_ENV=production vitest bench --config ./vitest.solid.config.ts", "test:perf:vue": "NODE_ENV=production vitest bench --config ./vitest.vue.config.ts", + "test:codspeed:react": "node ../../scripts/nx/run-codspeed-benchmark.mjs client-nav react", + "test:codspeed:solid": "node ../../scripts/nx/run-codspeed-benchmark.mjs client-nav solid", + "test:codspeed:vue": "node ../../scripts/nx/run-codspeed-benchmark.mjs client-nav vue", "test:types": "pnpm run test:types:react && pnpm run test:types:solid && pnpm run test:types:vue", "test:types:react": "tsc -p ./react/tsconfig.json --noEmit", "test:types:solid": "tsc -p ./solid/tsconfig.json --noEmit", @@ -45,7 +48,14 @@ "nx": { "targets": { "build:react": { - "cache": false, + "cache": true, + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/react/dist" + ], "dependsOn": [ { "projects": [ @@ -73,7 +83,14 @@ ] }, "build:solid": { - "cache": false, + "cache": true, + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/solid/dist" + ], "dependsOn": [ { "projects": [ @@ -101,7 +118,14 @@ ] }, "build:vue": { - "cache": false, + "cache": true, + "inputs": [ + "production", + "^production" + ], + "outputs": [ + "{projectRoot}/vue/dist" + ], "dependsOn": [ { "projects": [ @@ -172,6 +196,54 @@ "build:vue" ] }, + "test:codspeed:react": { + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ] + }, + "test:codspeed:solid": { + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ] + }, + "test:codspeed:vue": { + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ] + }, "test:types": { "cache": false, "dependsOn": [ diff --git a/benchmarks/memory/client/package.json b/benchmarks/memory/client/package.json index 2624b9e4e5..8f8f5fe668 100644 --- a/benchmarks/memory/client/package.json +++ b/benchmarks/memory/client/package.json @@ -40,7 +40,7 @@ "targets": { "build:react": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -57,7 +57,7 @@ }, "build:solid": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -74,7 +74,7 @@ }, "build:vue": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -227,6 +227,66 @@ "cwd": "benchmarks/memory/client" } }, + "test:codspeed:react": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-client react" + } + }, + "test:codspeed:solid": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-client solid" + } + }, + "test:codspeed:vue": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-client vue" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ diff --git a/benchmarks/memory/server/package.json b/benchmarks/memory/server/package.json index 68ba6b3c32..58f8587525 100644 --- a/benchmarks/memory/server/package.json +++ b/benchmarks/memory/server/package.json @@ -37,7 +37,7 @@ "targets": { "build:react": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -55,7 +55,7 @@ }, "build:solid": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -73,7 +73,7 @@ }, "build:vue": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -233,6 +233,66 @@ "cwd": "benchmarks/memory/server" } }, + "test:codspeed:react": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-server react" + } + }, + "test:codspeed:solid": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-server solid" + } + }, + "test:codspeed:vue": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs memory-server vue" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ diff --git a/benchmarks/ssr/package.json b/benchmarks/ssr/package.json index 40cd97df5e..6ebb14affd 100644 --- a/benchmarks/ssr/package.json +++ b/benchmarks/ssr/package.json @@ -28,7 +28,7 @@ "targets": { "build:react": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -54,7 +54,7 @@ }, "build:solid": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -80,7 +80,7 @@ }, "build:vue": { "executor": "nx:noop", - "cache": false, + "cache": true, "dependsOn": [ { "projects": [ @@ -150,6 +150,66 @@ "cwd": "benchmarks/ssr" } }, + "test:codspeed:react": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:react" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs ssr react" + } + }, + "test:codspeed:solid": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:solid" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs ssr solid" + } + }, + "test:codspeed:vue": { + "executor": "nx:run-commands", + "cache": true, + "inputs": [ + "default", + "^production", + { + "env": "GITHUB_RUN_ID" + }, + { + "env": "GITHUB_RUN_ATTEMPT" + } + ], + "dependsOn": [ + "build:vue" + ], + "options": { + "command": "node scripts/nx/run-codspeed-benchmark.mjs ssr vue" + } + }, "test:types": { "executor": "nx:noop", "dependsOn": [ diff --git a/scripts/nx/run-codspeed-benchmark.mjs b/scripts/nx/run-codspeed-benchmark.mjs new file mode 100644 index 0000000000..35ab0ac229 --- /dev/null +++ b/scripts/nx/run-codspeed-benchmark.mjs @@ -0,0 +1,265 @@ +import { createHash } from 'node:crypto' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { spawn } from 'node:child_process' +import { fileURLToPath } from 'node:url' + +const CODSPEED_VERSION = '4.17.0' +const CODSPEED_INSTALLER_SHA256 = + 'b5151c275a2b8055593cc20e7d3d0258059e07c8b633a7bb2519df9306f9fd67' + +const rootDir = dirname(dirname(dirname(fileURLToPath(import.meta.url)))) + +const benchmarkConfigs = { + 'client-nav': { + mode: 'simulation', + cwd: 'benchmarks/client-nav', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, + ssr: { + mode: 'simulation', + cwd: 'benchmarks/ssr', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, + 'memory-client': { + mode: 'memory', + cwd: 'benchmarks/memory/client', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, + 'memory-server': { + mode: 'memory', + cwd: 'benchmarks/memory/server', + frameworks: { + react: 'vitest.react.config.ts', + solid: 'vitest.solid.config.ts', + vue: 'vitest.vue.config.ts', + }, + }, +} + +const [benchmark, framework] = process.argv.slice(2) +const benchmarkConfig = benchmarkConfigs[benchmark] +const vitestConfig = benchmarkConfig?.frameworks[framework] + +if (!benchmarkConfig || !vitestConfig) { + console.error( + `Usage: node scripts/nx/run-codspeed-benchmark.mjs <${Object.keys( + benchmarkConfigs, + ).join('|')}> `, + ) + process.exit(1) +} + +await ensureGithubEventPath() + +const codspeed = await findCodSpeedBinary() +const codspeedArgs = [ + 'run', + `--mode=${process.env.CODSPEED_MODE ?? benchmarkConfig.mode}`, +] + +if (process.env.CODSPEED_TOKEN) { + codspeedArgs.push('--token', process.env.CODSPEED_TOKEN) +} + +codspeedArgs.push( + '--', + 'bash', + '-lc', + `NODE_ENV=production vitest bench --config ./${vitestConfig}`, +) + +await execFile(codspeed, codspeedArgs, { + cwd: join(rootDir, benchmarkConfig.cwd), + env: { + ...process.env, + WITH_INSTRUMENTATION: '1', + }, +}) + +async function ensureGithubEventPath() { + if (process.env.GITHUB_EVENT_PATH || process.env.GITHUB_ACTIONS !== 'true') { + return + } + + const eventDir = await mkdtemp(join(tmpdir(), 'codspeed-github-event-')) + const eventPath = join(eventDir, 'event.json') + const repository = process.env.GITHUB_REPOSITORY ?? '' + const repositoryId = Number.parseInt(process.env.GITHUB_REPOSITORY_ID ?? '0', 10) + const pullRequestNumber = getPullRequestNumber() + const event = { + repository: { + full_name: repository, + id: repositoryId, + private: true, + }, + ...(pullRequestNumber ? { number: pullRequestNumber } : {}), + pull_request: { + ...(pullRequestNumber ? { number: pullRequestNumber } : {}), + head: { + ref: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? '', + sha: process.env.GITHUB_SHA ?? '', + repo: { + full_name: repository, + id: repositoryId, + }, + }, + base: { + ref: process.env.GITHUB_BASE_REF ?? '', + repo: { + full_name: repository, + id: repositoryId, + }, + }, + }, + } + + await writeFile(eventPath, JSON.stringify(event), 'utf8') + process.env.GITHUB_EVENT_PATH = eventPath +} + +function getPullRequestNumber() { + const match = process.env.GITHUB_REF?.match(/^refs\/pull\/(\d+)\//) + + if (match) { + return Number.parseInt(match[1], 10) + } + + const refName = process.env.GITHUB_REF_NAME ?? '' + const refNameMatch = refName.match(/^(\d+)\//) + + if (refNameMatch) { + return Number.parseInt(refNameMatch[1], 10) + } + + return undefined +} + +async function findCodSpeedBinary() { + const path = await which('codspeed') + + if (path) { + return path + } + + await installCodSpeed() + + const installedPath = await which('codspeed', { + PATH: buildPathWithCodSpeedInstallDirs(), + }) + + if (installedPath) { + process.env.PATH = buildPathWithCodSpeedInstallDirs() + return installedPath + } + + throw new Error('Unable to find codspeed after installing the runner') +} + +async function installCodSpeed() { + const dir = await mkdtemp(join(tmpdir(), 'codspeed-runner-')) + const installer = join(dir, 'install.sh') + + try { + const body = await fetch( + `https://codspeed.io/v${CODSPEED_VERSION}/install.sh`, + ) + + if (!body.ok) { + throw new Error( + `Failed to download CodSpeed installer: HTTP ${body.status}`, + ) + } + + const buffer = Buffer.from(await body.arrayBuffer()) + const actualHash = createHash('sha256').update(buffer).digest('hex') + + if (actualHash !== CODSPEED_INSTALLER_SHA256) { + throw new Error( + `CodSpeed installer hash mismatch. Expected ${CODSPEED_INSTALLER_SHA256}, got ${actualHash}.`, + ) + } + + await writeFile(installer, buffer, { mode: 0o755 }) + await execFile('bash', [installer, '--quiet']) + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +async function which(binary, env = process.env) { + try { + const { stdout } = await execFile('bash', ['-lc', `command -v ${binary}`], { + env, + capture: true, + }) + + return stdout.trim() || undefined + } catch { + return undefined + } +} + +function buildPathWithCodSpeedInstallDirs() { + const home = process.env.HOME + + if (!home) { + return process.env.PATH ?? '' + } + + return [ + join(home, '.codspeed', 'bin'), + join(home, '.cargo', 'bin'), + join(home, '.local', 'bin'), + process.env.PATH ?? '', + ].join(':') +} + +async function execFile(file, args, options = {}) { + const child = spawn(file, args, { + cwd: options.cwd, + env: options.env, + stdio: options.capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', + }) + + let stdout = '' + let stderr = '' + + if (options.capture) { + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk) => { + stdout += chunk + }) + child.stderr.on('data', (chunk) => { + stderr += chunk + }) + } + + const code = await new Promise((resolve, reject) => { + child.on('error', reject) + child.on('close', resolve) + }) + + if (code !== 0) { + const message = options.capture + ? stderr || stdout || `${file} exited with code ${code}` + : `${file} exited with code ${code}` + throw new Error(message) + } + + return { stdout, stderr } +}