fix(ci): make releases/latest wait for a complete asset set

.github/actions/check-release-assets/action.yml

@@ -0,0 +1,170 @@
+# THE EXPECTED RELEASE-ASSET MATRIX, and the only place that knows it (dig-node#335).
+#
+# A `vX.Y.Z` release of dig-node has TWO independent consumers, and until #335 the release gate
+# only knew about one of them:
+#
+#   * dig-updater's feedsign resolves dig-node by the NATIVE INSTALL PACKAGE names — it hands a
+#     package to msiexec/installer/dpkg and never places a bare binary — and it fails CLOSED on the
+#     whole signed manifest when one component cannot be resolved (dig_ecosystem#2290).
+#   * dig-installer resolves BOTH `dig-node` and `dign` as RAW BINARIES through `releases/latest`
+#     (`dig-installer/src/release.rs:187`, stems at `:59` / `:78`). A release that is `latest`
+#     without those binaries makes every fresh install 404.
+#
+# The two sets are produced by two DIFFERENT workflows that finish at different times —
+# `package.yml` attaches the packages, `release.yml` attaches the binaries — so "the release
+# exists" and "the release is usable" are not the same statement. This action expresses the second
+# one, over both sets at once.
+#
+# It is a composite action rather than an inline `run:` block so that the SAME code path can be
+# driven from a literal asset list. That is what makes the guard falsifiable: the self-test in
+# `verify-release-assets.yml` feeds it a deliberately incomplete list on every PR and requires it
+# to fail. A guard that has never been observed going red is a hope, not a gate.
+name: Check release assets
+description: >-
+  Assert a dig-node release carries every asset its two consumers resolve — the native install
+  packages (dig-updater feedsign) and the raw dig-node/dign binaries (dig-installer). Polls,
+  because the packages and the binaries are attached by separate workflows.
+
+inputs:
+  tag:
+    description: "The release tag to check (e.g. v0.145.0)."
+    required: true
+  repo:
+    description: "OWNER/NAME of the repository holding the release."
+    required: false
+    default: ${{ github.repository }}
+  timeout_minutes:
+    description: "How long to wait for the assets to appear before failing."
+    required: false
+    default: "75"
+  assets:
+    description: >-
+      A newline-separated literal asset list to check INSTEAD of querying the release. For the
+      self-test only — when set, the action makes exactly one pass and never polls, so a
+      deliberately incomplete list fails immediately rather than burning the timeout.
+    required: false
+    default: ""
+  github_token:
+    description: "Token used to read the release. Unused when `assets` is supplied."
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Assert every consumer-resolvable asset is present
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.github_token }}
+        REPO: ${{ inputs.repo }}
+        TAG: ${{ inputs.tag }}
+        TIMEOUT_MINUTES: ${{ inputs.timeout_minutes }}
+        ASSETS_OVERRIDE: ${{ inputs.assets }}
+      run: |
+        set -euo pipefail
+
+        # The version as it appears in asset names: the tag without its leading `v`.
+        VERSION="${TAG#v}"
+
+        # Kept as a literal list rather than derived from a glob: a glob would happily accept a
+        # `.deb` for the wrong arch or a stale version and call the release complete, which is the
+        # failure this guard exists to catch.
+        #
+        # THIS IS ONE COPY OF A CROSS-REPO CONTRACT. Producers: `package.yml` (packages) and
+        # `build-binaries.yml` (binaries), both in this repo. Consumers:
+        # `dig-updater/crates/dig-updater-feedsign/src/resolve.rs` (`asset_name_parts`) and
+        # `dig-installer/src/release.rs`. Verifier: here. Nothing enforces that they agree, so they
+        # are held together by `SYSTEM.md` (dig-updater section, "dig-node release-asset file
+        # names") and the `canonical` skill. Change one, change all of them.
+        #
+        # macOS contributes ONE PACKAGE name, not two: the `.pkg` is universal and carries no arch
+        # token, so `macos/arm64` and `macos/x64` both resolve to it. The raw binaries below are
+        # per-arch and do carry the token.
+        #
+        # `arm64.deb` is required DELIBERATELY, and this is stricter than feedsign's own failure
+        # condition. feedsign fails closed only when a component resolves ZERO assets, so a release
+        # missing just `arm64.deb` would still publish — silently dropping linux/arm64 hosts from
+        # auto-update rather than reddening anything. That silent drop is exactly the arm64
+        # platform floor (dig_ecosystem#1741/#1736/#2126), so the stable channel treats a missing
+        # arm64 package as a failed release. Do not relax this to match feedsign.
+        PLATFORMS=(linux-arm64 linux-x64 macos-arm64 macos-x64 windows-x64.exe)
+
+        EXPECTED=(
+          # Native install packages — dig-updater feedsign.
+          "dig-node_${VERSION}_amd64.deb"
+          "dig-node_${VERSION}_arm64.deb"
+          "dig-node-${VERSION}-macos.pkg"
+          "dig-node-${VERSION}-windows-x64.msi"
+        )
+        # Raw binaries — dig-installer, which resolves the `dig-node` and `dign` stems separately
+        # and 404s on either one being absent. `dign` is not an optional extra: it is the CLI, and
+        # an install that lands the daemon without it leaves a node with no command
+        # (dig_ecosystem#857).
+        for p in "${PLATFORMS[@]}"; do
+          EXPECTED+=("dig-node-${VERSION}-${p}")
+          EXPECTED+=("dign-${VERSION}-${p}")
+        done
+
+        deadline=$(( $(date +%s) + TIMEOUT_MINUTES * 60 ))
+        attempt=0
+
+        while :; do
+          attempt=$(( attempt + 1 ))
+
+          if [ -n "$ASSETS_OVERRIDE" ]; then
+            # Self-test mode: one pass over a literal list, no polling and no network.
+            assets="$ASSETS_OVERRIDE"
+          else
+            # A missing release is a legitimate "not yet" while the release workflow is still
+            # running, so it is treated the same as a missing asset rather than aborting early.
+            #
+            # `select(.state == "uploaded")` IS LOAD-BEARING. GitHub creates the asset row when an
+            # upload STARTS, in state `starting`, so a name becomes visible before its bytes are.
+            # Only `uploaded` means the file can actually be downloaded.
+            #
+            # Without it this guard would NARROW the dig-node#335 race rather than close it. The
+            # calling job is `needs: publish`, which orders it after release.yml's OWN upload —
+            # but package.yml is a separate workflow with no ordering relationship to it at all.
+            # So the poll could observe all fourteen names while a `.msi` or `.pkg` was still
+            # uploading, and `promote` would move `latest` onto a release whose download is
+            # incomplete: the same observable failure as #335, through a shorter window.
+            #
+            # A `sleep` is NOT an acceptable substitute. It would shrink the window without
+            # removing it and could not be shown to fail, so the release would LOOK guarded while
+            # still racing. Ask for the state; do not wait and hope.
+            assets="$(gh release view "$TAG" --repo "$REPO" --json assets --jq '.assets[] | select(.state == "uploaded") | .name' 2>/dev/null || true)"
+          fi
+
+          missing=()
+          for name in "${EXPECTED[@]}"; do
+            printf '%s\n' "$assets" | grep -qxF "$name" || missing+=("$name")
+          done
+
+          if [ ${#missing[@]} -eq 0 ]; then
+            echo "$TAG carries all ${#EXPECTED[@]} consumer-resolvable assets — feedsign can resolve dig-node, and dig-installer can fetch dig-node + dign."
+            {
+              echo "### Release assets verified — \`$TAG\`"
+              echo
+              for name in "${EXPECTED[@]}"; do echo "- \`$name\`"; done
+            } >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          if [ -n "$ASSETS_OVERRIDE" ] || [ "$(date +%s)" -ge "$deadline" ]; then
+            {
+              echo "### Release assets MISSING — \`$TAG\`"
+              echo
+              echo "This release is NOT shippable. dig-updater feedsign fails closed on the whole"
+              echo "STABLE signed feed when a native package is absent, and dig-installer 404s on a"
+              echo "fresh install when a raw \`dig-node\`/\`dign\` binary is absent."
+              echo
+              echo "Missing:"
+              for name in "${missing[@]}"; do echo "- \`$name\`"; done
+            } >> "$GITHUB_STEP_SUMMARY"
+            echo "::error::release $TAG is missing ${#missing[@]} of ${#EXPECTED[@]} consumer-resolvable asset(s): ${missing[*]}. Attach them (dispatch package.yml and/or release.yml against the $TAG ref) before this release is allowed to stand as latest."
+            exit 1
+          fi
+
+          echo "attempt $attempt: still missing ${#missing[@]} of ${#EXPECTED[@]} (${missing[*]}); retrying…"
+          sleep 30
+        done

.github/workflows/package.yml

@@ -336,4 +336,10 @@ jobs:
           # Do NOT regenerate notes — release.yml (the binary release) owns the notes; this job
           # only appends the native-package assets to the same tag's release.
           generate_release_notes: false
+          # Nor does it own `latest` (dig-node#335). This job attaches FOUR of a stable release's
+          # fourteen assets; softprops promotes to `latest` by default, so on v0.145.0 this job
+          # made a release with no binaries at all the one dig-installer fetched from, five
+          # minutes before release.yml attached them. Promotion belongs to release.yml's
+          # `promote` job, which runs after the asset guard has confirmed BOTH publishers landed.
+          make_latest: "false"
           fail_on_unmatched_files: true

.github/workflows/release.yml

@@ -1,11 +1,13 @@
 # STABLE binary release. On a `vX.Y.Z` tag (cut by the nightly-release orchestrator's stable job
 # — nightly-release.yml — either from the midnight cron detecting a version bump or from a manual
 # `workflow_dispatch`), this workflow builds the `dig-node` service binary + its `dign` alias for
-# every OS/arch (via the reusable build workflow) and publishes them to a STABLE GitHub Release:
-# `prerelease: false`, marked `latest`. Every per-OS/arch binary is published under the canonical
-# `dig-node-*` name (+ the `dign-*` alias) — SPEC §11.2; the duplicate legacy `dig-companion-*`
-# copy was dropped in #585. The changelog is already
-# inside the tag (the orchestrator committed it before tagging), so the notes carry the changelog.
+# every OS/arch (via the reusable build workflow) and publishes them to a STABLE GitHub Release
+# with `prerelease: false`. It is then marked `latest` by the `promote` job at the bottom of this
+# file — never by the upload itself — and only after the asset guard has confirmed the release
+# carries everything both of its consumers resolve (dig-node#335). Every per-OS/arch binary is
+# published under the canonical `dig-node-*` name (+ the `dign-*` alias) — SPEC §11.2; the
+# duplicate legacy `dig-companion-*` copy was dropped in #585. The changelog is already inside the
+# tag (the orchestrator committed it before tagging), so the notes carry the changelog.
 #
 # This is intentionally tag-ONLY: merges to main no longer build or release here (dig_ecosystem
 # #590 batches releases to the nightly cron + manual dispatch). Pre-merge coverage comes from
@@ -80,11 +82,60 @@ jobs:
       - name: Create / update the STABLE release and attach binaries
         uses: softprops/action-gh-release@v2
         with:
-          # `prerelease: false` + `make_latest: true`: a stable release is the one that moves
-          # `latest`. Nightlies (nightly-release.yml) are always prerelease + never latest, so a
-          # nightly can never masquerade as this stable download.
+          # `prerelease: false` marks this a stable release. `make_latest: false` is DELIBERATE
+          # and is the dig-node#335 fix: attaching assets must not be what moves `latest`.
+          #
+          # A stable release is assembled by TWO workflows. This one attaches the binaries;
+          # package.yml attaches the native install packages. Whichever finished first used to
+          # promote the half-built release, and on v0.145.0 that was package.yml at 01:47:49Z —
+          # five minutes and three seconds before the binaries landed at 01:52:52Z. For that
+          # window `releases/latest` was a release with no `dig-node-*` or `dign-*` binary at all,
+          # and dig-installer resolves both stems through `releases/latest`, so every fresh
+          # install 404'd.
+          #
+          # Promotion now happens in the `promote` job below, gated on the asset guard, so
+          # `latest` can only ever name a release a user can actually install from. Nightlies
+          # (nightly-release.yml) remain prerelease + never latest, so a nightly still cannot
+          # masquerade as this stable download.
           prerelease: false
-          make_latest: "true"
+          make_latest: "false"
           files: release/*
           generate_release_notes: true
           fail_on_unmatched_files: true
+
+  # The release is complete only when BOTH publishers have finished, and this is the only job that
+  # knows when that is: the guard polls the published asset list until it holds every asset
+  # dig-updater feedsign and dig-installer resolve, or fails.
+  verify:
+    name: Verify the release is complete
+    needs: publish
+    if: github.ref_type == 'tag'
+    uses: ./.github/workflows/verify-release-assets.yml
+    with:
+      tag: ${{ github.ref_name }}
+
+  # PROMOTION IS THE LAST STEP, NOT A SIDE EFFECT OF UPLOADING (dig-node#335).
+  #
+  # `releases/latest` is a user-facing pointer: dig-installer fetches through it, so the moment it
+  # moves is the moment users are served that release. Moving it only after the guard has read the
+  # real asset list means an incomplete release is never `latest` — the previous complete release
+  # keeps serving installs, which is the correct failure mode. A release that never completes
+  # simply never gets promoted, and stays visible as a non-latest release for diagnosis.
+  promote:
+    name: Promote the release to latest
+    needs: verify
+    if: github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Mark the verified release as latest
+        shell: bash
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          TAG: ${{ github.ref_name }}
+        run: |
+          set -euo pipefail
+          gh release edit "$TAG" --repo "$REPO" --latest
+          echo "$TAG is now releases/latest — verified to carry every asset dig-installer and dig-updater resolve."