ci: automate releases with semantic-release

.github/workflows/release-notes-check.yml

@@ -0,0 +1,37 @@
+name: Release Notes Check
+
+# Guards the release toolchain: a floating dependency once dropped the
+# Features/Bug Fixes sections from the generated notes. This runs the pinned
+# semantic-release in dry-run mode against the real history and, when a release
+# is due, fails if the notes carry no sections. Scoped to the release
+# config/scripts so ordinary PRs are unaffected.
+on:
+  pull_request:
+    paths:
+      - .releaserc.mjs
+      - ci/release/**
+      - .github/workflows/release-notes-check.yml
+  push:
+    branches: [ main ]
+    paths:
+      - .releaserc.mjs
+      - ci/release/**
+      - .github/workflows/release-notes-check.yml
+
+permissions:
+  contents: read
+
+jobs:
+  release-notes-check:
+    name: Release Notes Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+      - name: Assert generated release notes contain the expected sections
+        run: ./ci/release/dry_run.sh

.github/workflows/release.yml

@@ -2,12 +2,11 @@ name: Release and Publish
 
 on:
   push:
-    branches: [ main ]
     tags: [ 'v[0-9]+.[0-9]+.[0-9]+' ]
 
 permissions:
   id-token: write
-  contents: write
+  contents: read
 
 jobs:
   build:
@@ -17,6 +16,8 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -33,18 +34,41 @@ jobs:
           name: dist
           path: dist/
           retention-days: 1
-  release:
-    name: Release to GitHub
+  github-release:
+    name: Attach assets to GitHub release
     runs-on: ubuntu-latest
     needs: build
-    if: startsWith(github.ref, 'refs/tags/v')
+    permissions:
+      contents: write
     steps:
       - name: Download artifact
         uses: actions/download-artifact@v8
         with:
           name: dist
           path: dist/
-      - name: Publish to GitHub release page
+      # semantic-release creates the release for this tag in the same run that
+      # pushed the tag, normally seconds before this build finishes. Wait for it
+      # so the upload below attaches to that existing release (with its generated
+      # notes) instead of racing ahead and creating a bare release -- which would
+      # in turn make semantic-release's own release step fail with a 409.
+      - name: Wait for the semantic-release GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          for attempt in $(seq 1 30); do
+            if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
+              echo "Release $TAG found."
+              exit 0
+            fi
+            echo "Release $TAG not present yet (attempt $attempt/30); waiting 10s..."
+            sleep 10
+          done
+          echo "::error::Release $TAG did not appear within 5 minutes; not creating a bare release." >&2
+          exit 1
+      # Omitting `body` leaves the generated notes untouched (the action retains
+      # existing release info on update); only the wheel/sdist are attached.
+      - name: Attach artifacts to the GitHub release
         uses: softprops/action-gh-release@v3
         with:
           files: |
@@ -53,7 +77,7 @@ jobs:
   publish:
     name: Publish to PyPI
     runs-on: ubuntu-latest
-    needs: release
+    needs: [build, github-release]
     environment: pypi
     steps:
       - name: Download artifact

.github/workflows/semantic-release.yml

@@ -0,0 +1,45 @@
+name: Semantic Release
+
+on:
+  schedule:
+    # 2 AM on Sunday
+    - cron: "0 2 * * 0"
+  workflow_dispatch:
+
+concurrency:
+  group: release
+  cancel-in-progress: false
+
+jobs:
+  semantic-release:
+    if: github.repository == 'substrait-io/substrait-python'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/create-github-app-token@v3
+        id: app-token
+        with:
+          client-id: ${{ secrets.RELEASER_ID }}
+          private-key: ${{ secrets.RELEASER_KEY }}
+      - name: Checkout code
+        uses: actions/checkout@v6
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+      - name: Get bot user ID
+        id: bot-user-id
+        run: |
+          echo "user-id=$(gh api "/users/${{ steps.app-token.outputs.app-slug }}[bot]" --jq .id)" >> "$GITHUB_OUTPUT"
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+      - name: Run semantic-release
+        run: ./ci/release/run.sh
+        env:
+          GITHUB_TOKEN: ${{ steps.app-token.outputs.token }}
+          GIT_AUTHOR_NAME: "${{ steps.app-token.outputs.app-slug }}[bot]"
+          GIT_COMMITTER_NAME: "${{ steps.app-token.outputs.app-slug }}[bot]"
+          GIT_AUTHOR_EMAIL: "${{ steps.bot-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com"
+          GIT_COMMITTER_EMAIL: "${{ steps.bot-user-id.outputs.user-id }}+${{ steps.app-token.outputs.app-slug }}[bot]@users.noreply.github.com"

.releaserc.mjs

@@ -0,0 +1,141 @@
+// semantic-release configuration.
+//
+// This is an ESM (.mjs) config rather than .releaserc.json for two reasons: the
+// release-notes-generator needs a `writerOpts.transform` function to strip git
+// trailers (e.g. `Signed-off-by:`) that the conventional-commits parser folds
+// into BREAKING CHANGE notes (JSON cannot carry functions), and the set of
+// plugins depends on RELEASE_DRY_RUN.
+//
+// The config defaults to a dry run as a fail-safe: the side-effecting plugins
+// (@semantic-release/github, @semantic-release/git) are only included when
+// RELEASE_DRY_RUN=false. ci/release/run.sh opts in to a real release that way;
+// every other invocation -- the credential-free notes check in
+// ci/release/dry_run.sh, or a forgotten/typo'd env var -- stays harmless.
+//
+// The env var is needed on top of --dry-run because --dry-run alone is not
+// enough: semantic-release still runs every plugin's verifyConditions step in
+// dry-run mode (it skips only prepare, publish, addChannel, success and fail).
+// @semantic-release/github's verifyConditions fails without a GITHUB_TOKEN, so
+// the side-effecting plugins must be omitted entirely, not merely guarded by
+// --dry-run, to allow a credential-free dry run.
+
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+
+// Load the conventionalcommits preset. The release scripts run semantic-release
+// via `npx -p ...`, which installs the preset as a sibling of semantic-release
+// in a temporary node_modules. A bare `import` here would resolve relative to
+// this config file's directory (the repo, which has no node_modules) and fail,
+// so fall back to resolving the preset relative to the running semantic-release
+// binary (process.argv[1]). The plain import path still covers local dev where
+// the preset is installed alongside the project.
+const loadPreset = async () => {
+  try {
+    return (await import("conventional-changelog-conventionalcommits")).default;
+  } catch {
+    const require = createRequire(pathToFileURL(process.argv[1]));
+    const resolved = pathToFileURL(
+      require.resolve("conventional-changelog-conventionalcommits"),
+    ).href;
+    return (await import(resolved)).default;
+  }
+};
+const conventionalcommits = await loadPreset();
+
+// Git trailers that should never appear in the changelog or release notes.
+const TRAILER_KEYS = [
+  "Signed-off-by",
+  "Co-authored-by",
+  "Co-developed-by",
+  "Reviewed-by",
+  "Acked-by",
+  "Tested-by",
+  "Reported-by",
+  "Suggested-by",
+  "Helped-by",
+  "Cc",
+];
+const TRAILER = new RegExp(`^(?:${TRAILER_KEYS.join("|")}):\\s`, "i");
+
+// The conventional-commits parser ends a BREAKING CHANGE note only at a
+// recognized reference (closes #..., fixes #...) or another note keyword, not
+// at a git trailer -- so a trailing `Signed-off-by:` gets absorbed into the
+// note text. Strip such trailing trailer lines.
+const stripTrailers = (text) => {
+  if (!text) return text;
+  const lines = text.split("\n");
+  while (lines.length) {
+    const last = lines[lines.length - 1].trim();
+    if (last === "" || TRAILER.test(last)) {
+      lines.pop();
+    } else {
+      break;
+    }
+  }
+  return lines.join("\n");
+};
+
+const preset = await conventionalcommits();
+const presetTransform = preset.writer.transform;
+
+// Dry run unless explicitly disabled (ci/release/run.sh sets RELEASE_DRY_RUN=false).
+const dryRun = process.env.RELEASE_DRY_RUN !== "false";
+
+export default {
+  branches: ["main"],
+  preset: "conventionalcommits",
+  dryRun,
+  plugins: [
+    [
+      "@semantic-release/commit-analyzer",
+      {
+        // Pre-1.0, matching substrait-java: a breaking change is a minor bump.
+        releaseRules: [{ breaking: true, release: "minor" }],
+      },
+    ],
+    [
+      "@semantic-release/release-notes-generator",
+      {
+        // Only `transform` is overridden; the generator merges this over the
+        // preset's writer options, so templates/grouping/sorting are kept.
+        writerOpts: {
+          transform(commit, context) {
+            const out = presetTransform(commit, context);
+            if (out && Array.isArray(out.notes)) {
+              out.notes = out.notes.map((note) => ({
+                ...note,
+                text: stripTrailers(note.text),
+              }));
+            }
+            return out;
+          },
+        },
+      },
+    ],
+    [
+      "@semantic-release/changelog",
+      {
+        changelogTitle: "Release Notes\n---",
+        changelogFile: "CHANGELOG.md",
+      },
+    ],
+    // GitHub releases and the release commit only happen in a real release.
+    ...(dryRun
+      ? []
+      : [
+          [
+            "@semantic-release/github",
+            {
+              successComment: false,
+            },
+          ],
+          [
+            "@semantic-release/git",
+            {
+              assets: ["CHANGELOG.md"],
+              message: "chore(release): ${nextRelease.version}",
+            },
+          ],
+        ]),
+  ],
+};

RELEASING.md

@@ -1,14 +1,61 @@
 # Releasing substrait-python
 
-Given that you are a Substrait committer or PMC member and have the appropriate permissions, releasing a new version of substrait-python is done by simply creating new Github Release through the UI:
+Releases of substrait-python are **fully automated** using
+[semantic-release](https://semantic-release.gitbook.io/) and follow the same model
+as the other Substrait projects (e.g. substrait-java).
 
-1. Go to https://github.com/substrait-io/substrait-python/releases
-2. Click `Draft a new release`
-3. Enter the version to be released in the `Tag` field prefixed with a lower case `v`, e.g. `v0.99.0`. Then click `Create new tag` which tells Github to create the tag on publication of the release.
-4. Click `Generate release notes` which will automatically populate the release title and release notes fields.
-5. If you are happy with the release notes and you are ready for an immediate release simply click `Publish release` otherwise save the release as a draft for later.
-6. Monitor the Github Actions release build for the newly created release and tag.
+The [Semantic Release](.github/workflows/semantic-release.yml) workflow runs on a
+weekly schedule (2 AM UTC on Sundays). It inspects the [Conventional
+Commits](https://www.conventionalcommits.org/en/v1.0.0/) merged since the last
+release, computes the next version, updates `CHANGELOG.md`, creates the `vX.Y.Z`
+git tag, and publishes a GitHub Release with auto-generated notes.
+
+Creating the tag triggers the [Release and Publish](.github/workflows/release.yml)
+workflow, which builds the package (the version is derived from the tag via
+`setuptools_scm`) and publishes it to [PyPI](https://pypi.org/project/substrait/)
+using Trusted Publishing.
+
+As a result, there is nothing to do by hand for a normal release — just make sure
+your PR titles/commit messages follow the Conventional Commits specification (this
+is enforced by the [PR Title Check](.github/workflows/pr_title.yml) workflow).
+
+## Triggering an off-cycle release
+
+If you need a release before the next scheduled run (and you are a Substrait
+committer or PMC member with the appropriate permissions):
+
+1. Go to https://github.com/substrait-io/substrait-python/actions/workflows/semantic-release.yml
+2. Click `Run workflow` and select the `main` branch.
+3. Monitor the workflow run, then the triggered `Release and Publish` run, to
+   confirm the new version reaches PyPI.
+
+If there are no release-worthy commits since the last release (only `chore`, `docs`,
+`ci`, etc.), semantic-release will report that no release is necessary and do
+nothing.
 
 ## Versioning
 
-substrait-python follows semantic versioning as described for the Substrait specification here: https://substrait.io/spec/versioning/.
+Version bumps are derived from commit types:
+
+| Commit type                                  | Version bump |
+| -------------------------------------------- | ------------ |
+| `fix:`                                       | patch        |
+| `feat:`                                      | minor        |
+| breaking change (`feat!:`, `BREAKING CHANGE`)| minor        |
+
+substrait-python follows semantic versioning as described for the Substrait
+specification here: https://substrait.io/spec/versioning/. Because the project is
+pre-1.0, breaking changes produce a **minor** bump rather than a major one (matching
+substrait-java).
+
+## Release toolchain
+
+The semantic-release toolchain versions are pinned in
+[`ci/release/run.sh`](ci/release/run.sh) and the release plugin set lives in
+[`.releaserc.mjs`](.releaserc.mjs). Pinning is deliberate: a floating
+`conventional-changelog-conventionalcommits` once dropped the `Features` /
+`Bug Fixes` sections from the generated notes. When bumping any of these
+versions, the [Release Notes Check](.github/workflows/release-notes-check.yml)
+workflow runs [`ci/release/dry_run.sh`](ci/release/dry_run.sh) — a credential-free
+dry run against the real history (in a throwaway worktree) that fails if a release
+is due but its notes come out with no sections.