actions: surface genuine oasdiff errors as PR annotations

.github/workflows/test-error-annotation.yaml

@@ -0,0 +1,66 @@
+name: error-annotation
+on:
+  pull_request:
+  push:
+jobs:
+  entrypoint_error_annotation:
+    runs-on: ubuntu-latest
+    name: Entrypoints surface genuine errors as annotations, not success/fail-on
+    steps:
+      - name: checkout
+        uses: actions/checkout@v6
+      - name: Exercise every entrypoint with a stubbed oasdiff
+        run: |
+          set -eu
+
+          # Stub oasdiff: write $STUB_STDERR to stderr (if set), produce no
+          # stdout, and exit $STUB_CODE. Lets us drive each entrypoint's
+          # exit-code handling deterministically — no real spec, no network.
+          mkdir -p /tmp/bin
+          cat > /tmp/bin/oasdiff <<'STUB'
+          #!/bin/sh
+          [ -n "${STUB_STDERR:-}" ] && printf '%s\n' "$STUB_STDERR" >&2
+          exit "${STUB_CODE:-0}"
+          STUB
+          chmod +x /tmp/bin/oasdiff
+          export PATH="/tmp/bin:$PATH"
+          export GITHUB_OUTPUT=/tmp/gh_output GITHUB_STEP_SUMMARY=/tmp/gh_summary
+          export GITHUB_REPOSITORY=o/r GITHUB_SHA=deadbeef GITHUB_REF=refs/pull/1/merge
+          echo '{}' > /tmp/event.json; export GITHUB_EVENT_PATH=/tmp/event.json
+
+          fail() { echo "FAIL: $1" >&2; exit 1; }
+
+          # invoke <exit-code> <stderr> <command> -> prints the entrypoint's stdout.
+          # pr-comment gets an empty oasdiff-token so it skips the network POST.
+          invoke() {
+            export STUB_CODE="$1" STUB_STDERR="$2"; cmd="$3"
+            : > "$GITHUB_OUTPUT"; : > "$GITHUB_STEP_SUMMARY"
+            case "$cmd" in
+              breaking)   sh breaking/entrypoint.sh   base.yaml rev.yaml "" "" "" "" "" "" "" "" "" "" "" "" "false" ;;
+              changelog)  sh changelog/entrypoint.sh  base.yaml rev.yaml "" "" "" "" "" "" "" "" "" "" "" "" "false" ;;
+              diff)       sh diff/entrypoint.sh       base.yaml rev.yaml "" "" "" "" "" "" "" "false" ;;
+              pr-comment) sh pr-comment/entrypoint.sh base.yaml rev.yaml "" "" "" "" "" "" "false" ;;
+              validate)   sh validate/entrypoint.sh   spec.yaml "" "false" ;;
+            esac
+          }
+
+          for cmd in breaking changelog diff pr-comment validate; do
+            # success (exit 0): no ::error:: annotation
+            out=$(invoke 0 "" "$cmd" 2>/dev/null || true)
+            echo "$out" | grep -q '::error::' && fail "$cmd: success (exit 0) must not emit ::error::"
+
+            # genuine error (exit 102): ::error:: annotation surfaced
+            out=$(invoke 102 'Error: failed to load spec from "x.yaml": no such file' "$cmd" 2>/dev/null || true)
+            echo "$out" | grep -q '::error::' || fail "$cmd: exit 102 must emit a ::error:: annotation"
+
+            # disallowed external ref (exit 123): the allow-external-refs remedy
+            out=$(invoke 123 'Error: external $ref not allowed: https://evil' "$cmd" 2>/dev/null || true)
+            echo "$out" | grep -q 'allow-external-refs: true' || fail "$cmd: exit 123 must emit the allow-external-refs remedy"
+
+            # fail-on / changes-found (exit 1, no stderr): no ::error:: annotation
+            out=$(invoke 1 "" "$cmd" 2>/dev/null || true)
+            echo "$out" | grep -q '::error::' && fail "$cmd: exit 1 (fail-on) must not emit ::error::"
+
+            echo "ok: $cmd"
+          done
+          echo "all error-annotation assertions passed"

breaking/entrypoint.sh

@@ -95,6 +95,12 @@ exit_code=0
 _err=$(mktemp)
 breaking_changes=$(oasdiff breaking "$base" "$revision" $flags $fail_on_flag 2>"$_err") || exit_code=$?
 [ -s "$_err" ] && cat "$_err" >&2
+# Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is
+# success and exit 1 is the intended "breaking changes found" / fail-on result;
+# only codes >=2 (load/parse/etc.) are real errors worth surfacing here.
+if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then
+    echo "::error::$(tr '\n' ' ' < "$_err")"
+fi
 # Exit code 123 = oasdiff refused a disallowed external $ref (stable contract,
 # not message text). Surface the action-specific remedy.
 if [ "$exit_code" -eq 123 ]; then

changelog/entrypoint.sh

@@ -104,6 +104,11 @@ else
 fi
 if [ "$exit_code" -ne 0 ]; then
     [ -s "$_err" ] && cat "$_err" >&2
+    # Promote a genuine failure to a Checks-tab annotation. Exit 1 is the
+    # intended fail-on result (not an error); only codes >=2 are real errors.
+    if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then
+        echo "::error::$(tr '\n' ' ' < "$_err")"
+    fi
     # Exit code 123 = oasdiff refused a disallowed external $ref (stable
     # contract, not message text). Surface the action-specific remedy.
     if [ "$exit_code" -eq 123 ]; then

diff/entrypoint.sh

@@ -88,6 +88,12 @@ else
     output=$(oasdiff diff "$base" "$revision" 2>"$_err") || exit_code=$?
 fi
 [ -s "$_err" ] && cat "$_err" >&2
+# Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is
+# success and exit 1 is the intended fail-on-diff result; only codes >=2
+# (load/parse/etc.) are real errors worth surfacing here.
+if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then
+    echo "::error::$(tr '\n' ' ' < "$_err")"
+fi
 # Exit code 123 = oasdiff refused a disallowed external $ref (stable contract,
 # not message text). Surface the action-specific remedy.
 if [ "$exit_code" -eq 123 ]; then

pr-comment/entrypoint.sh

@@ -46,6 +46,11 @@ _err=$(mktemp)
 changelog=$(oasdiff changelog "$base" "$revision" --format json $flags 2>"$_err") || oasdiff_exit=$?
 if [ "$oasdiff_exit" -ne 0 ] && [ -z "$changelog" ]; then
     [ -s "$_err" ] && cat "$_err" >&2
+    # Promote a genuine failure to a Checks-tab annotation. Exit 1 is the
+    # intended fail-on result (not an error); only codes >=2 are real errors.
+    if [ "$oasdiff_exit" -ge 2 ] && [ -s "$_err" ]; then
+        echo "::error::$(tr '\n' ' ' < "$_err")"
+    fi
     # Exit code 123 = oasdiff refused a disallowed external $ref (stable
     # contract, not message text). Surface the action-specific remedy.
     if [ "$oasdiff_exit" -eq 123 ]; then

validate/entrypoint.sh

@@ -35,6 +35,12 @@ exit_code=0
 _err=$(mktemp)
 oasdiff validate $flags --format githubactions "$spec" 2>"$_err" || exit_code=$?
 [ -s "$_err" ] && cat "$_err" >&2
+# Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is
+# success and exit 1 is the intended fail-on result; only codes >=2
+# (load/parse/etc.) are real errors worth surfacing here.
+if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then
+    echo "::error::$(tr '\n' ' ' < "$_err")"
+fi
 # Exit code 123 = oasdiff refused a disallowed external $ref (stable contract,
 # not message text). Surface the action-specific remedy.
 if [ "$exit_code" -eq 123 ]; then