From a8749316dc75b4ff3202eabf270e0f20c067a57b Mon Sep 17 00:00:00 2001 From: Ashraf Fouda Date: Mon, 27 Jul 2026 17:45:23 +0300 Subject: [PATCH] fix(upgrade): deliver canary version only to test_farms The update worker now holds the network `latest` symlink at the last GA version during a canary (safe_to_upgrade = false), so the upgrader can no longer assume the symlink equals the version the node should run. Compute the target version from the chain and, for a node whose farm is in test_farms during a canary, retarget the taglink at the chain version tag directly (resolvable on the hub independently of the latest symlink) instead of the held-back symlink. Non-canary and freshly bootstrapped nodes follow the GA symlink; once safe_to_upgrade flips true and the worker advances the symlink, everyone rolls forward. Co-Authored-By: Claude Opus 4.8 --- pkg/upgrade/upgrade.go | 50 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/pkg/upgrade/upgrade.go b/pkg/upgrade/upgrade.go index 289c6346..71e7359b 100644 --- a/pkg/upgrade/upgrade.go +++ b/pkg/upgrade/upgrade.go @@ -281,13 +281,6 @@ func (u *Upgrader) update(ctx context.Context) error { return errors.Wrap(err, "failed to get remote tag") } - // obviously a remote tag need to match the current tag. - // if the remote is different, we actually run the update and exit. - if remote.Target == current.Target { - // nothing to do! - return nil - } - env := environment.MustGet() gw := stubs.NewSubstrateGatewayStub(u.zcl) chainVer, testFarms, err := getRolloutConfig(ctx, gw) @@ -295,25 +288,36 @@ func (u *Upgrader) update(ctx context.Context) error { return errors.Wrap(err, "failed to get rollout config and version") } - remoteVer := remote.Target[strings.LastIndex(remote.Target, "/")+1:] + // the version the chain wants this node to run + targetVer := chainVer.Version if kernel.GetParams().IsLight() { - if env.RunningMode != environment.RunningDev && (remoteVer != chainVer.VersionLight) { - // nothing to do! hub version is not the same as the chain - return nil - } - } else { - if env.RunningMode != environment.RunningDev && (remoteVer != chainVer.Version) { - // nothing to do! hub version is not the same as the chain - return nil - } + targetVer = chainVer.VersionLight } - if !chainVer.SafeToUpgrade { - if !slices.Contains(testFarms, uint32(env.FarmID)) { - // nothing to do! waiting for the flag `safe to upgrade to be enabled after A/B testing` - // node is not a part of A/B testing - return nil - } + // During a canary rollout (safe_to_upgrade == false) the update worker holds the + // network `latest` taglink back at the last GA version, so a freshly bootstrapped + // node comes up on GA and non-canary nodes stay put. Only nodes whose farm is in + // test_farms take part, and they must target the chain (canary) version directly + // because `latest` no longer points at it. + if !chainVer.SafeToUpgrade && slices.Contains(testFarms, uint32(env.FarmID)) { + // retarget the taglink at the chain version, keeping the repo/tags prefix + prefix := remote.Target[:strings.LastIndex(remote.Target, "/")+1] + remote.Target = prefix + targetVer + } + + // obviously a remote tag needs to differ from the current tag. + // if the remote is different, we actually run the update and exit. + if remote.Target == current.Target { + // nothing to do! + return nil + } + + // the resolved hub version must match the version the chain wants us to run; for a + // non-canary node during a canary this keeps it from moving ahead of the GA `latest`. + remoteVer := remote.Target[strings.LastIndex(remote.Target, "/")+1:] + if env.RunningMode != environment.RunningDev && remoteVer != targetVer { + // nothing to do! `latest` hasn't caught up to the chain version yet + return nil } log.Info().Str("running version", u.Version().String()).Str("updating to version", filepath.Base(remote.Target)).Msg("updating system...")