diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 76f81c9..32f325a 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -63,6 +63,10 @@ on: description: 'Fail the build on gosec findings. SARIF is uploaded either way.' type: boolean default: true + check-nested-modules: + description: 'Check that nested go.mod files require their siblings at versions the module proxy can serve. A nested module replace is ignored once the module is somebody else dependency, so a require naming an unpublished version breaks go get for every consumer while every local build keeps working. No-op in a repo with a single go.mod.' + type: boolean + default: true secrets: CODECOV_TOKEN: required: false @@ -444,4 +448,116 @@ jobs: echo "Failing because govulncheck-fail-on-findings is true." exit 1 fi - echo "govulncheck-fail-on-findings is false; reporting only." + echo "govulncheck-fail-on-findings is false; reporting only." + + nested-modules: + name: Nested modules + runs-on: ubuntu-latest + if: inputs.check-nested-modules + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version: ${{ inputs.primary-go-version || '1.25' }} + cache: false + + - name: Check intra-repo requires resolve + shell: bash + run: | + # Check that every intra-repo require in a nested module names a version a + # consumer can actually resolve. + # + # A nested module carries a replace so the working tree builds against + # itself. That replace is ignored the moment the module becomes somebody + # else's dependency, so the require line is what a consumer resolves. If it + # names a version that was never published, `go get` on that nested module + # fails outright, and no ordinary build notices because the replace hides it. + # + # default mode fails on v0.0.0 and on anything that is not valid semver. + # A version that simply is not published yet is reported and + # tolerated, because release rewrites these before tagging. + # --strict additionally requires every version to be live on the + # proxy. Run this after tagging, when it must all be true. + set -uo pipefail + + STRICT=0 + [ "${1:-}" = "--strict" ] && { STRICT=1; shift; } + cd "${1:-.}" + + ROOT=$(awk '/^module /{print $2; exit}' go.mod 2>/dev/null || true) + if [ -z "$ROOT" ]; then + echo "no root go.mod, nothing to check" + exit 0 + fi + + # The proxy encodes an uppercase letter as !x. xraph paths are lowercase but + # the encoding is part of the protocol, so do it properly. + escape() { printf '%s' "$1" | sed -E 's/([A-Z])/!\L\1/g'; } + + fail=0 + checked=0 + pending=0 + # Portable memo of proxy answers. An associative array would be neater but + # needs bash 4, and on bash 3.2 it fails in a way that silently undercounts + # instead of erroring, which is worse than being a little verbose here. + seen_ok="" + seen_missing="" + while IFS= read -r f; do + [ "$f" = "./go.mod" ] && continue + dir=${f%/go.mod}; dir=${dir#./} + while read -r path ver; do + [ -z "$path" ] && continue + checked=$((checked + 1)) + + # v0.0.0 and the zero pseudo-version are the two shapes Go leaves + # behind when a module is only ever reached through a replace. The + # second is valid semver, which is exactly why it slips past a + # format check, and it resolves for nobody. + if [ "$ver" = "v0.0.0" ] || [ "$ver" = "v0.0.0-00010101000000-000000000000" ]; then + echo "::error file=${f#./}::$path is required at $ver, which is not a version anyone can fetch. Someone running 'go get $path' gets 'unknown revision v0.0.0'. It works for you only because you already require $ROOT at a real version. Name a released version here; the replace directive keeps local builds on the working tree." + fail=1 + continue + fi + if ! printf '%s' "$ver" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$'; then + echo "::error file=${f#./}::$path is required at '$ver', which is not a semver version." + fail=1 + continue + fi + + # Most nested modules in a repo require the same root version, so + # ask the proxy once per distinct pair rather than once per module. + key="${path}@${ver}" + case "$seen_ok" in + *"|${key}|"*) continue ;; + esac + case "$seen_missing" in + *"|${key}|"*) ;; + *) + if curl -sSfL --max-time 20 "https://proxy.golang.org/$(escape "$path")/@v/${ver}.info" > /dev/null 2>&1; then + seen_ok="${seen_ok}|${key}|" + continue + fi + seen_missing="${seen_missing}|${key}|" + ;; + esac + if [ "$STRICT" -eq 1 ]; then + echo "::error file=${f#./}::$path@$ver is not on the module proxy, so nobody can consume $dir." + fail=1 + else + echo "::notice file=${f#./}::$path@$ver is not published yet. Fine before a first release, since release pins these to the tag it cuts, but it has to be real by then." + pending=$((pending + 1)) + fi + done < <(go mod edit -json "$f" 2>/dev/null | jq -r --arg r "$ROOT" '.Require[]? | select(.Path == $r or (.Path | startswith($r + "/"))) | "\(.Path) \(.Version)"') + done < <(find . -name go.mod -not -path './.git/*' -not -path '*/node_modules/*' -not -path './.claude/*' -not -path '*/testdata/*' -not -path '*/vendor/*' -not -path '*/_*' | sort) + + if [ "$checked" -eq 0 ]; then + echo "no nested module requires anything from this repo, nothing to check" + elif [ "$fail" -eq 0 ]; then + echo "checked $checked intra-repo require(s), all resolvable ($pending not published yet)" + fi + exit "$fail" diff --git a/.github/workflows/go-release.yml b/.github/workflows/go-release.yml index db21c14..f5b07b8 100644 --- a/.github/workflows/go-release.yml +++ b/.github/workflows/go-release.yml @@ -15,9 +15,17 @@ on: type: string default: '' submodules: - description: 'JSON array of package suffixes within the root module, e.g. ["errs","log"] (like go-utils). NOT for repos with genuinely nested go.mod files - those need their own per-submodule tags (e.g. discovery/v1.2.0), which this workflow does not create.' + description: 'JSON array of package suffixes within the root module, e.g. ["errs","log"] (like go-utils). These are packages, not modules: they ship inside the root module and only need naming so the release notes mention them. A directory with its own go.mod is a different thing and belongs in nested-modules, which tags it separately.' type: string default: '[]' + nested-modules: + description: 'JSON array of directories holding their own go.mod, e.g. ["drivers/pgdriver","extension"]. Empty means discover them. Each is released by tagging /, which is how the proxy finds a version for a module that is not at the repo root. Set to "[]" and skip-nested-modules to opt out.' + type: string + default: '' + skip-nested-modules: + description: 'Do not touch nested go.mod files at all.' + type: boolean + default: false doc-links: description: 'JSON array of repo-relative doc paths, e.g. ["errs/README.md"]' type: string @@ -226,6 +234,128 @@ jobs: git tag -a "$VERSION" -m "Release $VERSION" git push origin "$VERSION" + - name: Discover nested modules + id: nested + if: ${{ !inputs.skip-nested-modules }} + shell: bash + env: + NESTED_IN: ${{ inputs.nested-modules }} + run: | + set -euo pipefail + if [ -n "$NESTED_IN" ]; then + LIST=$(printf '%s' "$NESTED_IN" | jq -r '.[]') + else + LIST=$(find . -name go.mod \ + -not -path './.git/*' -not -path '*/node_modules/*' \ + -not -path './.claude/*' -not -path '*/testdata/*' \ + -not -path '*/vendor/*' -not -path '*/_*' \ + | sed 's|^\./||; s|/go\.mod$||' | grep -v '^go\.mod$' | sort) + fi + { + echo "list<> "$GITHUB_OUTPUT" + if [ -z "$LIST" ]; then + echo "No nested modules." >> "$GITHUB_STEP_SUMMARY" + else + echo "### Nested modules" >> "$GITHUB_STEP_SUMMARY" + printf '%s\n' "$LIST" | while IFS= read -r m; do + [ -n "$m" ] && echo "- $m" >> "$GITHUB_STEP_SUMMARY" + done + fi + + - name: Verify nested modules + if: ${{ !inputs.skip-nested-modules && inputs.run-tests }} + shell: bash + env: + LIST: ${{ steps.nested.outputs.list }} + run: | + set -euo pipefail + for mod in $LIST; do + echo "::group::$mod" + ( cd "$mod" && go mod download && go build ./... && go test -race -count=1 ./... ) + echo "::endgroup::" + done + + - name: Pin and tag nested modules + if: ${{ !inputs.skip-nested-modules }} + shell: bash + env: + VERSION: ${{ steps.ver.outputs.version }} + LIST: ${{ steps.nested.outputs.list }} + run: | + set -euo pipefail + [ -z "$LIST" ] && exit 0 + ROOT=$(awk '/^module /{print $2; exit}' go.mod) + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # A nested module carries a replace so the working tree builds + # against itself, and that replace is ignored the moment the module + # is somebody else dependency. The require line is what a consumer + # resolves, so on a branch it names the previous release and here it + # has to name the one being cut. Pin, commit, tag the commit. + # + # The commit is never pushed to a branch. It is reachable only + # through these tags, so the branch keeps pointing nested modules at + # the last release and keeps building against the working tree. + # go mod edit rather than a regex: it understands both the + # parenthesised block and the single-line require form, it never + # touches a replace, and it refuses a version whose major does not + # match the path suffix. A sed would quietly accept all three. + for mod in $LIST; do + paths=$(go mod edit -json "$mod/go.mod" \ + | jq -r --arg r "$ROOT" '.Require[]? | select(.Path == $r or (.Path | startswith($r + "/"))) | .Path') + for path in $paths; do + go mod edit -require="${path}@${VERSION}" "$mod/go.mod" + done + git add "$mod/go.mod" + done + if [ -n "$(git diff --cached --name-only)" ]; then + git commit -q -m "chore: pin nested modules to $VERSION" + fi + + for mod in $LIST; do + MOD_TAG="$mod/$VERSION" + if git rev-parse -q --verify "refs/tags/$MOD_TAG" >/dev/null; then + echo "$MOD_TAG already exists, skipping" + continue + fi + git tag -a "$MOD_TAG" -m "Release $MOD_TAG" + git push origin "$MOD_TAG" + echo "tagged $MOD_TAG" + done + + - name: Verify nested modules resolve from the proxy + if: ${{ !inputs.skip-nested-modules }} + shell: bash + env: + VERSION: ${{ steps.ver.outputs.version }} + LIST: ${{ steps.nested.outputs.list }} + run: | + set -euo pipefail + [ -z "$LIST" ] && exit 0 + ROOT=$(awk '/^module /{print $2; exit}' go.mod) + # The failure this catches is a nested module whose require names a + # version the proxy cannot serve. Every local build hides it behind + # the replace, so the first person to run go get finds it instead. + PROBE=$(mktemp -d) + trap 'rm -rf "$PROBE"' EXIT + cd "$PROBE" + go mod init example.com/release-probe >/dev/null + rc=0 + for mod in $LIST; do + echo "::group::go get $ROOT/$mod@$VERSION" + if ! GOFLAGS=-mod=mod GOPROXY=https://proxy.golang.org,direct \ + go get "$ROOT/$mod@$VERSION"; then + echo "::error::$ROOT/$mod@$VERSION does not resolve for a consumer." + rc=1 + fi + echo "::endgroup::" + done + exit "$rc" + - name: Assemble release notes id: notes shell: bash @@ -235,6 +365,7 @@ jobs: SUBMODULES: ${{ inputs.submodules }} DOC_LINKS: ${{ inputs.doc-links }} CHANGELOG_BODY: ${{ steps.changelog.outputs.body }} + NESTED_LIST: ${{ steps.nested.outputs.list }} run: | set -euo pipefail MODULE="$MODULE_IN" @@ -251,6 +382,9 @@ jobs: echo "go get ${MODULE}@${VERSION}" printf '%s' "$SUBMODULES" | jq -r --arg m "$MODULE" --arg v "$VERSION" \ '.[] | "go get \($m)/\(.)@\($v)"' + for nested in $NESTED_LIST; do + echo "go get ${MODULE}/${nested}@${VERSION}" + done echo '```' echo echo '## Changes' @@ -285,9 +419,13 @@ jobs: VERSION: ${{ steps.ver.outputs.version }} MODULE: ${{ steps.notes.outputs.module }} SUBMODULES: ${{ inputs.submodules }} + NESTED_LIST: ${{ steps.nested.outputs.list }} run: | set -euo pipefail curl -sSf "https://proxy.golang.org/${MODULE}/@v/${VERSION}.info" || true for sub in $(printf '%s' "$SUBMODULES" | jq -r '.[]'); do curl -sSf "https://proxy.golang.org/${MODULE}/${sub}/@v/${VERSION}.info" || true done + for nested in $NESTED_LIST; do + curl -sSf "https://proxy.golang.org/${MODULE}/${nested}/@v/${VERSION}.info" || true + done