feat: certify npm releases across operating systems [policy]

heimanba · heimanba · commit ae7721c433fe · 2026-07-16T19:49:31.000+08:00
Change-Id: I1826efe4bf8457db7fd976b779aade310a8d2c2b
diff --git a/.changeset/calm-spiders-test.md b/.changeset/calm-spiders-test.md
@@ -0,0 +1,7 @@
+---
+"@openagentpack/sdk": minor
+"@openagentpack/cli": minor
+"@openagentpack/playground": minor
+---
+
+Require maintained Node.js releases (22 or newer) and certify published packages on Linux, Windows, and macOS before creating a GitHub Release.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -121,7 +121,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        node: [20, 24]
+        node: [22, 24]
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
       - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
diff --git a/.github/workflows/compatibility-canary.yml b/.github/workflows/compatibility-canary.yml
@@ -0,0 +1,34 @@
+name: Package compatibility canary
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "41 5 * * 3"
+
+concurrency:
+  group: package-compatibility-canary
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  node-current:
+    name: npm latest / ${{ matrix.os }} / Node 26
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.5"
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: "26"
+          package-manager-cache: false
+      - run: npm install --global npm@12.0.1
+      - name: Install and execute npm latest
+        run: bun scripts/release/consumer-smoke.ts smoke --version latest
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -22,13 +22,14 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  publish:
+  preflight:
     if: vars.NPM_RELEASE_ENABLED == 'true' && inputs.confirm == 'PUBLISH'
     runs-on: ubuntu-latest
-    environment: npm-release
+    outputs:
+      channel: ${{ steps.release.outputs.channel }}
+      version: ${{ steps.release.outputs.version }}
     permissions:
-      contents: write
-      id-token: write
+      contents: read
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
         with:
@@ -63,14 +64,46 @@ jobs:
       - name: Verify release
         run: bun run verify:release
 
+  publish:
+    needs: preflight
+    runs-on: ubuntu-latest
+    environment: npm-release
+    permissions:
+      contents: write
+      id-token: write
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          fetch-depth: 0
+
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.5"
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: "24"
+          registry-url: https://registry.npmjs.org
+          package-manager-cache: false
+
+      - name: Install trusted-publishing npm CLI
+        run: |
+          npm install --global npm@12.0.1
+          npm --version
+
+      - name: Build publishable packages
+        run: |
+          bun install --frozen-lockfile
+          bun run build:packages
+
       - name: Publish packages with npm Trusted Publishing
         env:
           NPM_CONFIG_PROVENANCE: "true"
         run: bun run release:publish
 
       - name: Create immutable Git tag
         env:
-          VERSION: ${{ steps.release.outputs.version }}
+          VERSION: ${{ needs.preflight.outputs.version }}
         run: |
           tag="v${VERSION}"
           if git rev-parse --verify --quiet "refs/tags/${tag}"; then
@@ -85,11 +118,62 @@ jobs:
             git push origin "${tag}"
           fi
 
+  registry-ready:
+    needs: [preflight, publish]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.5"
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: "24"
+          package-manager-cache: false
+      - run: npm install --global npm@12.0.1
+      - name: Wait for the fixed release group to reach npm
+        run: bun scripts/release/consumer-smoke.ts wait --version "${{ needs.preflight.outputs.version }}"
+
+  post-release-consumer:
+    name: Consumer / ${{ matrix.os }} / Node ${{ matrix.node }}
+    needs: [preflight, registry-ready]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        node: [22, 24]
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: "1.3.5"
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: ${{ matrix.node }}
+          package-manager-cache: false
+      - run: npm install --global npm@12.0.1
+      - name: Install and execute the public packages
+        run: bun scripts/release/consumer-smoke.ts smoke --version "${{ needs.preflight.outputs.version }}"
+
+  finalize-release:
+    needs: [preflight, publish, post-release-consumer]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          fetch-depth: 0
       - name: Create GitHub Release
         env:
           GH_TOKEN: ${{ github.token }}
-          VERSION: ${{ steps.release.outputs.version }}
-          CHANNEL: ${{ steps.release.outputs.channel }}
+          VERSION: ${{ needs.preflight.outputs.version }}
+          CHANNEL: ${{ needs.preflight.outputs.channel }}
         run: |
           tag="v${VERSION}"
           if gh release view "${tag}" >/dev/null 2>&1; then
diff --git a/docs/contributing/release.md b/docs/contributing/release.md
@@ -41,7 +41,7 @@ Choose the SemVer impact and describe the change for the generated changelog. On
 5. Select `release/0.1.0-beta` in the workflow branch dropdown, choose `beta`, type `PUBLISH`, and run it.
 6. Approve the `npm-release` Environment deployment after reviewing the commit and job summary.
 
-The publish workflow validates that the selected branch, package version, and channel agree. It publishes with the npm `beta` dist-tag, creates an immutable `v0.1.0-beta.N` tag, and creates a GitHub prerelease.
+The publish workflow validates that the selected branch, package version, and channel agree. It publishes with the npm `beta` dist-tag and creates the immutable `v0.1.0-beta.N` tag. It then installs that exact version from the public npm registry on Linux, Windows, and macOS under Node.js 22 and 24. The GitHub prerelease is created only after all six consumer jobs pass.
 
 For another beta, merge fixes and their changesets into `main`, then rerun **Prepare Beta Release** for the same series. The workflow merges `main` into the beta branch and calculates the next beta. Never merge the beta branch back into `main`; delete it after the stable release.
 
@@ -52,15 +52,29 @@ For another beta, merge fixes and their changesets into `main`, then rerun **Pre
 3. Select `main`, choose `stable`, type `PUBLISH`, and run it.
 4. Approve the `npm-release` Environment deployment after reviewing the exact package version.
 
-Only a clean `X.Y.Z` version on `main` can publish to npm's `latest` tag. A successful run creates the matching immutable Git tag and GitHub Release. Publishing is idempotent: packages whose exact version already exists are skipped, so a partially failed run can be retried.
+Only a clean `X.Y.Z` version on `main` can publish to npm's `latest` tag. Publishing creates the matching immutable Git tag, waits for all three packages to become visible in the public registry, and runs the six-job consumer matrix. The GitHub Release is the final certification step and is created only after that matrix passes. Publishing is idempotent: packages whose exact version already exists are skipped, so a partially failed publish can be retried from the same commit.
+
+## Post-release consumer verification
+
+The release is not considered complete when `npm publish` returns successfully. The workflow must also prove that a clean external consumer can install and execute the public packages from npm:
+
+| Runner | Required Node.js versions |
+|---|---|
+| Ubuntu | 22, 24 |
+| Windows | 22, 24 |
+| macOS | 22, 24 |
+
+Each matrix job installs the exact version independently, verifies npm registry signatures and provenance, loads every public SDK entry point, runs CLI help/version/offline validation, and starts the Playground over HTTP. It does not consume workspace packages, local tarballs, or build artifacts from the publish job.
+
+The `Package compatibility canary` workflow also installs npm's `latest` release on all three operating systems under Node.js 26 every Wednesday. Canary failures do not affect an existing release, but they should be triaged before Node.js 26 becomes part of the supported LTS matrix.
 
 ## Local verification
 
 ```bash
 bun run verify:release
 ```
 
-This runs the full repository checks, builds the packages, performs `npm publish --dry-run`, and installs the packed artifacts into clean Node.js 20 and 24 consumer projects. To inspect only the package tarballs:
+This runs the full repository checks, builds the packages, performs a registry-independent `npm pack --dry-run`, and installs the packed artifacts as a clean external consumer. CI runs that package smoke under Node.js 22 and 24. To inspect only the package tarballs:
 
 ```bash
 bun run build:packages
@@ -91,5 +105,7 @@ After installing the CLI, run `agents --help`. A beta user returns to stable wit
 
 - If npm authentication fails, compare the Trusted Publisher repository, workflow filename, and Environment character-for-character with the table above.
 - If a package published before another failed, rerun the same workflow from the same commit. Already-published exact versions are skipped.
+- If all packages published but a post-release consumer job fails, keep the immutable tag, do not unpublish or move the tag, and do not create the GitHub Release. Fix the compatibility issue and publish a new patch version; npm package versions cannot be overwritten.
+- Registry visibility is retried for five minutes before it is classified as a release failure. Retry the same workflow only when npm propagation, rather than package compatibility, was the cause.
 - If a version tag already points at another commit, stop. Tags are immutable; investigate the repository history instead of moving or deleting the tag.
 - If **Prepare Beta Release** reports no unreleased changesets, add a changeset on `main` before preparing another beta.
diff --git a/docs/contributing/release.zh-CN.md b/docs/contributing/release.zh-CN.md
@@ -41,7 +41,7 @@ bun run changeset
 5. 在 workflow branch 下拉框选择 `release/0.1.0-beta`，channel 选择 `beta`，输入 `PUBLISH`，然后运行。
 6. 检查提交和 job 信息后，批准 `npm-release` Environment deployment。
 
-发布工作流会同时校验分支、包版本和 channel。通过后使用 npm 的 `beta` dist-tag 发布，创建不可变的 `v0.1.0-beta.N` Git tag，并创建 GitHub prerelease。
+发布工作流会同时校验分支、包版本和 channel。通过后使用 npm 的 `beta` dist-tag 发布，并创建不可变的 `v0.1.0-beta.N` Git tag。随后，它会在 Linux、Windows、macOS 的 Node.js 22 和 24 环境中，从公共 npm registry 安装该精确版本。只有六个消费者 job 全部通过，才创建 GitHub prerelease。
 
 需要下一个 Beta 时，把修复及其 changeset 合并到 `main`，再对同一个版本系列运行 **Prepare Beta Release**。工作流会把 `main` 合入 Beta 分支并计算下一个 Beta。不要把 Beta 分支反向合并到 `main`；稳定版发布后删除它。
 
@@ -52,15 +52,29 @@ bun run changeset
 3. 选择 `main`，channel 选择 `stable`，输入 `PUBLISH`，然后运行。
 4. 检查准确版本后，批准 `npm-release` Environment deployment。
 
-只有 `main` 上不含预发布后缀的 `X.Y.Z` 才能发布到 npm 的 `latest`。成功后会创建对应的不可变 Git tag 和正式 GitHub Release。发布过程可安全重试：已存在的精确包版本会被跳过。
+只有 `main` 上不含预发布后缀的 `X.Y.Z` 才能发布到 npm 的 `latest`。发布后会创建对应的不可变 Git tag，等待三个包在公共 registry 中全部可见，再执行六个消费者 job。正式 GitHub Release 是最后的发布认证，只有矩阵全部通过后才创建。发布过程可安全重试：已经存在的精确包版本会被跳过。
+
+## 发布后消费者验证
+
+`npm publish` 返回成功并不代表 release 已完成。工作流还必须证明一个全新的外部消费者可以从 npm 安装并执行公开包：
+
+| Runner | 必测 Node.js 版本 |
+|---|---|
+| Ubuntu | 22、24 |
+| Windows | 22、24 |
+| macOS | 22、24 |
+
+每个矩阵 job 都会独立安装精确版本，验证 npm registry 签名和 provenance，加载 SDK 的所有公开入口，执行 CLI help/version/离线 validate，并通过 HTTP 启动 Playground。验证过程不会使用 workspace 包、本地 tarball 或 publish job 的构建产物。
+
+`Package compatibility canary` 工作流还会在每周三使用三个操作系统和 Node.js 26 验证 npm 的 `latest` 版本。Canary 失败不会影响已经完成的 release，但应在 Node.js 26 进入正式 LTS 支持矩阵前完成处置。
 
 ## 本地验证
 
 ```bash
 bun run verify:release
 ```
 
-它会运行完整仓库检查、构建包、执行 `npm publish --dry-run`，并把 tarball 安装进干净的 Node.js 20/24 消费者项目。只检查 tarball 时可以运行：
+它会运行完整仓库检查、构建包、执行不依赖 registry 版本状态的 `npm pack --dry-run`，并以全新外部消费者的方式安装 tarball。CI 会在 Node.js 22 和 24 下执行这项 package smoke。只检查 tarball 时可以运行：
 
 ```bash
 bun run build:packages
@@ -91,5 +105,7 @@ npm install @openagentpack/sdk
 
 - npm 认证失败：逐字检查 Trusted Publisher 的仓库、workflow 文件名和 Environment 是否与上表一致。
 - 部分包成功、部分包失败：从同一个提交重新运行同一个工作流，已发布的精确版本会被跳过。
+- 所有包都已发布，但发布后消费者 job 失败：保留不可变 tag，不执行 unpublish、不移动 tag，也不创建 GitHub Release。修复兼容性问题后发布新的 patch 版本；npm 上的版本不能被覆盖。
+- registry 可见性会重试五分钟，之后才判定 release 失败。只有确认失败原因是 npm 同步延迟而不是包兼容性时，才从同一个提交重试。
 - 版本 tag 已指向其他提交：立即停止。tag 必须保持不可变，应排查历史，不能移动或删除 tag。
 - **Prepare Beta Release** 提示没有未发布 changeset：先在 `main` 添加 changeset，再准备下一个 Beta。
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -20,7 +20,7 @@
 		"url": "https://github.com/modelstudioai/OpenAgentPack/issues"
 	},
 	"engines": {
-		"node": ">=20"
+		"node": ">=22"
 	},
 	"type": "module",
 	"main": "./dist/src/program.js",
diff --git a/packages/playground/package.json b/packages/playground/package.json
@@ -19,7 +19,7 @@
 		"url": "https://github.com/modelstudioai/OpenAgentPack/issues"
 	},
 	"engines": {
-		"node": ">=20"
+		"node": ">=22"
 	},
 	"type": "module",
 	"bin": {
diff --git a/packages/sdk/package.json b/packages/sdk/package.json
@@ -20,7 +20,7 @@
 		"url": "https://github.com/modelstudioai/OpenAgentPack/issues"
 	},
 	"engines": {
-		"node": ">=20"
+		"node": ">=22"
 	},
 	"type": "module",
 	"main": "./dist/index.js",
diff --git a/scripts/open-source.test.ts b/scripts/open-source.test.ts
@@ -52,6 +52,15 @@ describe("open-source repository invariants", () => {
 		expect(changesets.fixed).toContainEqual(manifests.map((manifest) => manifest.name));
 	});
 
+	test("public packages require a maintained Node.js baseline", () => {
+		for (const pkg of ["sdk", "playground", "cli"]) {
+			const manifest = JSON.parse(readFileSync(resolve(root, `packages/${pkg}/package.json`), "utf8")) as {
+				engines?: { node?: string };
+			};
+			expect(manifest.engines?.node).toBe(">=22");
+		}
+	});
+
 	test("published packages have no runtime dependency on private workspaces", () => {
 		const packageFiles = publicWorktreeFilesMatching(
 			(file) => isVisiblePath(file) && (file === "package.json" || file.endsWith("/package.json")),
@@ -140,6 +149,15 @@ describe("open-source repository invariants", () => {
 		expect(workflow).toContain("workflow_dispatch:");
 		expect(workflow).toContain('git config user.name "github-actions[bot]"');
 		expect(workflow).toContain("github-actions[bot]@users.noreply.github.com");
+		expect(workflow).toContain("registry-ready:");
+		expect(workflow).toContain("post-release-consumer:");
+		expect(workflow).toContain("os: [ubuntu-latest, windows-latest, macos-latest]");
+		expect(workflow).toContain("node: [22, 24]");
+		expect(workflow).toContain("needs: [preflight, publish, post-release-consumer]");
+		const consumerGate = workflow.indexOf("post-release-consumer:");
+		const finalize = workflow.indexOf("finalize-release:");
+		expect(consumerGate).toBeGreaterThan(-1);
+		expect(finalize).toBeGreaterThan(consumerGate);
 	});
 
 	test("public worktree has no high-confidence secrets or internal machine references", () => {
diff --git a/scripts/release/consumer-smoke.test.ts b/scripts/release/consumer-smoke.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, test } from "bun:test";
+import { commonRegistryVersion, packageName, registryVersion } from "./consumer-smoke.ts";
+
+describe("published consumer smoke", () => {
+	test("builds canonical package names", () => {
+		expect(packageName("sdk")).toBe("@openagentpack/sdk");
+	});
+
+	test("reads an exact registry version", () => {
+		expect(registryVersion('"1.2.3"', "@openagentpack/sdk")).toBe("1.2.3");
+		expect(() => registryVersion("[]", "@openagentpack/sdk")).toThrow("invalid version");
+	});
+
+	test("requires the fixed release group to resolve one version", () => {
+		expect(
+			commonRegistryVersion([
+				{ name: "@openagentpack/sdk", version: "1.2.3" },
+				{ name: "@openagentpack/cli", version: "1.2.3" },
+			]),
+		).toBe("1.2.3");
+		expect(() =>
+			commonRegistryVersion([
+				{ name: "@openagentpack/sdk", version: "1.2.3" },
+				{ name: "@openagentpack/cli", version: "1.2.4" },
+			]),
+		).toThrow("versions must match");
+	});
+});
diff --git a/scripts/release/consumer-smoke.ts b/scripts/release/consumer-smoke.ts
diff --git a/scripts/release/publish.test.ts b/scripts/release/publish.test.ts
diff --git a/scripts/release/publish.ts b/scripts/release/publish.ts
