Add protocol version auto-update workflow

.github/workflows/auto-update-protocol-version.yml

@@ -0,0 +1,133 @@
+name: Auto Update Protocol Version
+
+on:
+  schedule:
+  - cron: '30 1 * * *'   # 5:30 PM Pacific
+  workflow_dispatch:
+
+jobs:
+
+  check:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      has_updates: ${{ steps.changes.outputs.found }}
+    steps:
+    - uses: actions/checkout@v4
+
+    # Resolve inherits and every dep ref to a commit SHA up front, in separate
+    # steps each printing its input/diff/output, so the protocol values and the
+    # SHAs a PR records come from the same commits (a tracked branch could
+    # otherwise move between steps) and each stage is easy to debug.
+    - name: Images Input
+      run: |
+        echo "Input:"
+        < images.json jq -C
+        echo "images=$(< images.json jq -c)" >> $GITHUB_ENV
+
+    - name: Images with Inherits Resolved
+      run: |
+        images_before="$images"
+        images="$(<<< "$images" ./.scripts/images-resolve-inherits)"
+        echo "Diff:"
+        diff -u --color=always <(<<< "$images_before" jq) <(<<< "$images" jq) || true
+        echo "Output:"
+        <<< "$images" jq -C
+        echo "images=$images" >> $GITHUB_ENV
+
+    - name: Images with Extras
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        images_before="$images"
+        images="$(<<< "$images" ./.scripts/images-with-extras)"
+        echo "Diff:"
+        diff -u --color=always <(<<< "$images_before" jq) <(<<< "$images" jq) || true
+        echo "Output:"
+        <<< "$images" jq -C
+        echo "$images" > images.resolved.json
+
+    # Only images with config.autoupdate_protocol_version_default are updated.
+    - name: Update protocol_version_default
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: ./.scripts/auto-update-protocol-version images.resolved.json
+
+    - name: Check for changes
+      id: changes
+      run: |
+        if git diff --quiet images.json; then
+          echo "found=false" >> $GITHUB_OUTPUT
+        else
+          echo "found=true" >> $GITHUB_OUTPUT
+        fi
+
+    - name: Upload artifact
+      if: steps.changes.outputs.found == 'true'
+      uses: actions/upload-artifact@v4
+      with:
+        name: images
+        path: |
+          images.json
+          images.resolved.json
+
+  create-pr:
+    runs-on: ubuntu-latest
+    needs: check
+    if: needs.check.outputs.has_updates == 'true'
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: images
+
+    - name: Create Pull Request
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        # Key the branch on the resolved snapshot (commit SHAs) computed in the
+        # check job. A downstream component change yields a new hash -> a new
+        # PR, instead of deduping onto the existing one.
+        resolved=$(cat images.resolved.json)
+        hash=$(printf '%s' "$resolved" | sha256sum | cut -c1-16)
+        branch="auto-update-protocol-version/${hash}"
+
+        # Skip only if a PR for this exact component set is already open.
+        if gh pr list --head "$branch" --state open --json number --jq '.[0].number' | grep -q .; then
+          echo "PR already exists for this update"
+          exit 0
+        fi
+
+        git config user.name "github-actions[bot]"
+        git config user.email "github-actions[bot]@users.noreply.github.com"
+        git checkout -b "$branch"
+        git add images.json
+        git commit -m "Update protocol_version_default (${hash})"
+        git push -u origin "$branch"
+
+        {
+          echo "This PR was automatically generated by the auto-update-protocol-version workflow."
+          echo
+          echo "<details>"
+          echo "<summary>Resolved images.json</summary>"
+          echo
+          echo '~~~json'
+          echo "$resolved"
+          echo '~~~'
+          echo
+          echo "</details>"
+        } > pr-body.md
+
+        gh pr create \
+          --title "Update protocol_version_default (${hash})" \
+          --body-file pr-body.md \
+          --base main \
+          --head "$branch"
+
+        gh pr merge --auto --squash "$branch"

.scripts/auto-update-protocol-version

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+
+import json
+import re
+import subprocess
+import sys
+
+# Auto-update script for the protocol_version_default field in images.json
+#
+# Usage: ./.scripts/auto-update-protocol-version <resolved-images.json> [image-tag ...]
+#
+# For each image with config.autoupdate_protocol_version_default true (all such by
+# default, or only those named as arguments), sets config.protocol_version_default
+# to the highest ledger protocol version supported by ALL of its xdr, core, rpc and
+# horizon components. Images with it false (e.g. 'future') are left for manual
+# updates.
+#
+#   xdr      rs-stellar-xdr   Cargo.toml [package] version's major
+#   core     stellar-core     src/main/Config.cpp CURRENT_LEDGER_PROTOCOL_VERSION
+#   rpc      stellar-rpc      go.mod's go-stellar-sdk -> its Makefile XDR_COMMIT
+#   horizon  stellar-horizon  go.mod's go-stellar-sdk -> its Makefile XDR_COMMIT
+#
+# rpc and horizon don't state a protocol version of their own, so theirs is derived
+# from the XDR they build against: the stellar-xdr commit their go-stellar-sdk
+# generates from, compared with the stellar-xdr commit core pins at
+# src/protocol-curr/xdr. Matching or newer means they handle core's protocol and so
+# carry core's version; older means they can't and the image is left alone.
+
+def main():
+    args = sys.argv[1:]
+    if not args:
+        print("Usage: auto-update-protocol-version <resolved-images.json> [image-tag ...]",
+              file=sys.stderr)
+        sys.exit(1)
+    resolved_path, tags = args[0], args[1:]
+
+    with open('images.json', 'r') as f:
+        images = json.load(f)
+    with open(resolved_path, 'r') as f:
+        resolved = json.load(f)
+    by_tag = {image['tag']: image for image in images}
+
+    # Compute against the resolved commit SHAs, then write the resulting
+    # protocol_version_default into images.json (refs untouched).
+    updated = False
+    for src in resolved:
+        tag = src['tag']
+        if tags and tag not in tags:
+            continue
+        if not src['config'].get('autoupdate_protocol_version_default', True):
+            print(f"{tag}: skipping, autoupdate_protocol_version_default is false")
+            continue
+        if update_image(src):
+            by_tag[tag]['config']['protocol_version_default'] = \
+                src['config']['protocol_version_default']
+            updated = True
+
+    if updated:
+        with open('images.json', 'w') as f:
+            json.dump(images, f, indent=2)
+            f.write('\n')
+
+    # Sync the resolved snapshot so it records the exact state the PR proposes:
+    # the resolved commit SHAs plus the resulting protocol_version_default.
+    with open(resolved_path, 'w') as f:
+        json.dump(resolved, f, indent=2)
+        f.write('\n')
+
+def update_image(image):
+    """Set image's protocol_version_default to min over its components. Returns True if changed."""
+    tag = image['tag']
+    deps = {dep['name']: dep for dep in image['deps']}
+    for name in ('xdr', 'core', 'rpc', 'horizon'):
+        if name not in deps:
+            print(f"{tag}: skipping, no '{name}' dep")
+            return False
+
+    core = require(tag, 'core', deps['core'], core_version)
+    supported = {'xdr': require(tag, 'xdr', deps['xdr'], xdr_version), 'core': core}
+
+    # rpc and horizon are at core's protocol as long as the XDR they build against
+    # is not older than the XDR core pins.
+    core_xdr = require(tag, 'core', deps['core'], core_xdr_commit)
+    for name in ('rpc', 'horizon'):
+        dep = deps[name]
+        commit = require(tag, name, dep, sdk_xdr_commit)
+        status = xdr_compare(core_xdr, commit)
+        if status is None:
+            print(f"{tag}: failed to compare {name}'s xdr with core's", file=sys.stderr)
+            sys.exit(1)
+        if status not in ('identical', 'ahead'):
+            print(f"{tag}: skipping, {name}'s xdr {commit[:12]} is {status} "
+                  f"vs core's {core_xdr[:12]}")
+            return False
+        supported[name] = core
+
+    version = min(supported.values())
+    current = image['config'].get('protocol_version_default')
+    if current == version:
+        print(f"{tag}: protocol_version_default matches components ({version}) {supported}")
+        return False
+
+    print(f"{tag}: protocol_version_default {current} -> {version} {supported}")
+    image['config']['protocol_version_default'] = version
+    return True
+
+def require(tag, name, dep, extract):
+    """extract(dep), or exit: a read/parse failure must not leave the image stale."""
+    value = extract(dep)
+    if value is None:
+        print(f"{tag}: failed to read protocol version from {name} ({dep['repo']}@{dep['ref']})",
+              file=sys.stderr)
+        sys.exit(1)
+    return value
+
+def xdr_version(dep):
+    """Major of the [package] version in rs-stellar-xdr's Cargo.toml."""
+    text = read_source(dep, 'Cargo.toml')
+    if text is None:
+        return None
+    section = None
+    for line in text.splitlines():
+        stripped = line.strip()
+        if stripped.startswith('[') and stripped.endswith(']'):
+            section = stripped[1:-1]
+        elif section == 'package':
+            match = re.match(r'version\s*=\s*"(\d+)', stripped)
+            if match:
+                return int(match.group(1))
+    return None
+
+def core_version(dep):
+    """CURRENT_LEDGER_PROTOCOL_VERSION from stellar-core."""
+    text = read_source(dep, 'src/main/Config.cpp')
+    if text is None:
+        return None
+    match = re.search(r'CURRENT_LEDGER_PROTOCOL_VERSION\s*=\s*(\d+)', text)
+    if not match:
+        return None
+    version = int(match.group(1))
+    # stellar-core built with this flag supports one version beyond its source
+    # constant, matching the #ifdef in Config.cpp; read it from the core dep options.
+    next_flag = '--enable-next-protocol-version-unsafe-for-production'
+    if next_flag in dep.get('options', {}).get('configure_flags', ''):
+        version += 1
+    return version
+
+def core_xdr_commit(dep):
+    """stellar-xdr commit that stellar-core pins as its protocol XDR."""
+    return read_submodule(dep, 'src/protocol-curr/xdr')
+
+def sdk_xdr_commit(dep):
+    """stellar-xdr commit the dep's go-stellar-sdk generates its XDR from."""
+    text = read_source(dep, 'go.mod')
+    if text is None:
+        return None
+    match = re.search(r'github\.com/stellar/go-stellar-sdk\s+(\S+)', text)
+    if not match:
+        return None
+    text = read_file('stellar/go-stellar-sdk', module_ref(match.group(1)), 'Makefile')
+    if text is None:
+        return None
+    match = re.search(r'^XDR_COMMIT\s*=\s*(\w+)', text, re.MULTILINE)
+    return match.group(1) if match else None
+
+def module_ref(version):
+    """Git ref for a go module version: a pseudo-version's commit, else the tag."""
+    match = re.search(r'-([0-9a-f]{12})$', version)
+    return match.group(1) if match else version
+
+def xdr_compare(base, head):
+    """How head compares with base in stellar-xdr: identical, ahead, behind, diverged."""
+    result = subprocess.run(
+        ['gh', 'api', f'repos/stellar/stellar-xdr/compare/{base}...{head}', '--jq', '.status'],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        print(f"Error: failed to compare stellar-xdr {base}...{head}: {result.stderr}",
+              file=sys.stderr)
+        return None
+    return result.stdout.strip() or None
+
+def read_source(dep, path):
+    """Read a file from the dep's repo at its resolved sha, or None."""
+    return read_file(dep['repo'], dep['sha'], path)
+
+def read_file(repo, ref, path):
+    """Read a file from a repo at a ref via the GitHub API, or None."""
+    result = subprocess.run(
+        ['gh', 'api', '-H', 'Accept: application/vnd.github.raw',
+         f"repos/{repo}/contents/{path}?ref={ref}"],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        print(f"Error: failed to read {path} from {repo}@{ref}: {result.stderr}",
+              file=sys.stderr)
+        return None
+    return result.stdout
+
+def read_submodule(dep, path):
+    """Commit a submodule is pinned at in the dep's repo, or None."""
+    result = subprocess.run(
+        ['gh', 'api', f"repos/{dep['repo']}/contents/{path}?ref={dep['sha']}", '--jq', '.sha'],
+        capture_output=True, text=True
+    )
+    if result.returncode != 0:
+        print(f"Error: failed to read submodule {path} from {dep['repo']}@{dep['ref']}: {result.stderr}",
+              file=sys.stderr)
+        return None
+    return result.stdout.strip() or None
+
+if __name__ == '__main__':
+    main()

images.json

@@ -3,7 +3,8 @@
     "tag": "latest",
     "events": ["pull_request", "push"],
     "config": {
-      "protocol_version_default": 28
+      "protocol_version_default": 28,
+      "autoupdate_protocol_version_default": true
     },
     "deps": [
       {
@@ -66,7 +67,8 @@
     "tag": "testing",
     "events": ["pull_request", "push"],
     "config": {
-      "protocol_version_default": 28
+      "protocol_version_default": 28,
+      "autoupdate_protocol_version_default": true
     },
     "deps": [
       {
@@ -135,6 +137,7 @@
     "events": ["pull_request", "push"],
     "config": {
       "protocol_version_default": 28,
+      "autoupdate_protocol_version_default": false,
       "horizon_skip_protocol_version_check": true
     },
     "deps": [
@@ -192,7 +195,8 @@
     "tag": "nightly",
     "events": ["push", "schedule"],
     "config": {
-      "protocol_version_default": 28
+      "protocol_version_default": 28,
+      "autoupdate_protocol_version_default": true
     },
     "deps": [
       {
@@ -250,6 +254,7 @@
     "events": ["push", "schedule"],
     "config": {
       "protocol_version_default": 28,
+      "autoupdate_protocol_version_default": true,
       "horizon_skip_protocol_version_check": true
     },
     "deps": [