[PRODENG-3642] Validate pod CIDR against the live Swarm overlay address pool

.github/workflows/smoke-tests.yaml

@@ -130,3 +130,27 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: make smoke-upgrade
+
+  # PRODENG-3642. Deliberately omits the `push` trigger and the broad
+  # `smoke-test` label: this spends a stack on validation logic that changes
+  # rarely, so it is run when that logic is touched rather than every merge.
+  smoke-swarm-pool:
+    runs-on: arc-runner-set-mirantis-public
+    if: contains(github.event.pull_request.labels.*.name, 'smoke-swarm-pool')
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v7
+      - name: Set up Go
+        uses: actions/setup-go@v7
+        with:
+          go-version-file: go.mod
+          cache: true
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v4
+        with:
+          terraform_wrapper: false
+      - name: Run swarm address pool smoke test
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: make smoke-swarm-pool

Makefile

@@ -69,6 +69,9 @@ smoke-fips:
 .PHONY: smoke-upgrade
 smoke-upgrade:
 	go test -count=1 -v ./test/smoke/... -run TestUpgrade -timeout 90m
+.PHONY: smoke-swarm-pool
+smoke-swarm-pool:
+	go test -count=1 -v ./test/smoke/... -run TestSwarmAddrPoolCluster -timeout 60m
 .PHONY: clean-launchpad-chart
 clean-launchpad-chart:
 	terraform -chdir=./examples/tf-aws/launchpad apply --auto-approve --destroy

docs/usage/swarm-overlay-address-pool.md

@@ -0,0 +1,98 @@
+# Swarm overlay address pool
+
+Swarm allocates the subnets for overlay networks — including `ingress` — from its
+default address pool. Unless a pool was given when the swarm was created, that
+pool is `10.0.0.0/8` with a 24-bit subnet mask, so `ingress` becomes `10.0.0.0/24`.
+
+## The pool cannot be changed after the swarm is created
+
+`--default-addr-pool` is accepted only by `docker swarm init`. It is not part of
+the swarm specification that `docker swarm update` modifies, so there is no
+command that changes the pool of a running swarm:
+
+```
+$ docker swarm update --default-addr-pool 10.10.0.0/16
+unknown flag: --default-addr-pool
+```
+
+Launchpad passes `spec.mcr.swarmInstallFlags` to `docker swarm init`, which means
+those flags take effect only on the run that creates the swarm. On any later run
+against an existing cluster they are ignored, and launchpad logs a warning
+naming them.
+
+Setting `--default-addr-pool` on a cluster whose swarm already exists therefore
+changes nothing on the hosts. Launchpad reports this as a warning:
+
+```
+mcr.swarmInstallFlags sets --default-addr-pool 10.0.0.0/16 but the existing swarm
+allocates overlay networks from 10.0.0.0/8. A swarm's address pool is fixed when
+the swarm is created and cannot be changed on a running cluster, so this setting
+has no effect here and the cluster no longer matches its configuration.
+```
+
+## Reading the pool a cluster is using
+
+On a manager:
+
+```bash
+docker info --format '{{.Swarm.LocalNodeState}}|{{if .Swarm.Cluster}}{{range .Swarm.Cluster.DefaultAddrPool}}{{.}} {{end}}{{end}}'
+```
+
+An empty pool list means the swarm was created without `--default-addr-pool` and
+is using the `10.0.0.0/8` default. The subnet mask length is
+`{{.Swarm.Cluster.SubnetSize}}`.
+
+The `ingress` network is allocated from the pool and confirms it independently:
+
+```bash
+docker network inspect ingress --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}'
+```
+
+## Overlap with the Kubernetes pod CIDR
+
+`--pod-cidr` in `spec.mke.installFlags` must not overlap the Swarm pool.
+An overlap can leave the container runtime with a broken network configuration
+during MKE bootstrap, which drops the SSH connection and surfaces as a
+connection timeout after 20 or more minutes.
+
+Launchpad checks this in the `Validate Facts` phase, and what it does depends on
+whether the conflict is still avoidable:
+
+| Cluster state | Pool compared against | On overlap |
+|---|---|---|
+| No swarm yet | `spec.mcr.swarmInstallFlags`, or `10.0.0.0/8` | Fails. Choose a non-overlapping `--pod-cidr`, or set `--default-addr-pool`, which will be applied. |
+| Swarm exists | The pool read from the swarm | Warns and continues. The pool cannot be changed, so failing would only block upgrades of clusters already running this way. |
+
+## Changing the pool on an existing cluster
+
+There is no non-destructive procedure. The swarm must be dissolved and
+re-created, which **destroys every overlay network and every service running on
+them**, including `ingress`. Plan a maintenance window.
+
+If the goal is only to resolve a pod CIDR overlap, changing `--pod-cidr` instead
+is far less disruptive and is the recommended option.
+
+To change the pool:
+
+1. Record the current configuration — `docker network ls`, and
+   `docker network inspect` for each overlay network you will need to re-create.
+2. Stop workloads that depend on overlay networking.
+3. On every node, leave the swarm: `docker swarm leave --force`.
+4. Set the desired pool in the cluster configuration:
+
+   ```yaml
+   spec:
+     mcr:
+       swarmInstallFlags:
+         - --default-addr-pool=10.10.0.0/16
+         - --default-addr-pool-mask-length=24
+   ```
+
+5. Run `launchpad apply`. With no swarm present, `InitSwarm` creates one and the
+   flags are applied.
+6. Confirm the new pool with the `docker info` command above, and check that
+   `ingress` has been allocated from it.
+7. Re-create the overlay networks and redeploy the workloads recorded in step 1.
+
+`--default-addr-pool` may be repeated to give the swarm more than one pool.
+Launchpad validates `--pod-cidr` against all of them.

pkg/product/common/config/mcr_config.go

@@ -60,6 +60,11 @@ type MCRMetadata struct {
 	ManagerJoinToken string
 	WorkerJoinToken  string
 	MCRChannel       string
+	// SwarmDefaultAddrPool holds the overlay address pools an already existing
+	// swarm allocates from, as discovered on the swarm leader. It is empty when
+	// no swarm exists yet, so a non-empty value means the swarm predates this
+	// run and its pool can no longer be changed. See PRODENG-3642.
+	SwarmDefaultAddrPool []string
 }
 
 // UnmarshalYAML puts in sane defaults when unmarshaling from yaml.

pkg/product/mke/phase/gather_facts.go

@@ -60,6 +60,17 @@ func (p *GatherFacts) Run() error {
 			log.Infof("%s: MKE is not installed", swarmLeader)
 		}
 		p.Config.Spec.MKE.Metadata.ClusterID = swarm.ClusterID(swarmLeader)
+
+		// The overlay address pool of an existing swarm is needed by
+		// ValidateFacts, which runs before MCR is installed and so cannot read
+		// it from a host itself. A read failure is not fatal: validation falls
+		// back to the configured pool.
+		pools, err := swarm.DefaultAddrPool(swarmLeader)
+		if err != nil {
+			log.Warnf("%s: failed to read the swarm overlay address pool: %s", swarmLeader, err.Error())
+		} else if p.Config.Spec.MCR.Metadata != nil {
+			p.Config.Spec.MCR.Metadata.SwarmDefaultAddrPool = pools
+		}
 	}
 	if p.Config.Spec.ContainsMSR() {
 		// If we intend to configure msr as well, gather facts for msr

pkg/product/mke/phase/init_swarm.go

@@ -45,7 +45,12 @@ func (p *InitSwarm) Run() error {
 	} else {
 		log.Infof("%s: swarm already initialized", swarmLeader)
 		if len(p.Config.Spec.MCR.SwarmInstallFlags) > 0 {
-			log.Warnf("%s: swarm install flags ignored due to swarm cluster already existing", swarmLeader)
+			// Naming the flags matters: these are settings the operator believes
+			// are in force, and several of them (--default-addr-pool in
+			// particular) cannot be applied to a swarm after it exists.
+			// See PRODENG-3642.
+			log.Warnf("%s: swarm install flags only apply when a swarm is created, so %q has no effect on this cluster",
+				swarmLeader, p.Config.Spec.MCR.SwarmInstallFlags.Join())
 		}
 	}
 

pkg/product/mke/phase/validate_facts.go

@@ -4,11 +4,14 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"slices"
 	"strconv"
+	"strings"
 
 	"github.com/Mirantis/launchpad/pkg/mke"
 	"github.com/Mirantis/launchpad/pkg/phase"
 	mkeconfig "github.com/Mirantis/launchpad/pkg/product/mke/config"
+	"github.com/Mirantis/launchpad/pkg/swarm"
 	"github.com/hashicorp/go-version"
 	log "github.com/sirupsen/logrus"
 )
@@ -70,6 +73,8 @@ func (p *ValidateFacts) Run() error {
 		}
 	}
 
+	p.warnSwarmAddrPoolDivergence()
+
 	if err := p.validatePodCIDR(); err != nil {
 		return errors.Join(ErrFactsArentValid, err)
 	}
@@ -203,16 +208,67 @@ func (p *ValidateFacts) validateDataPlane() error {
 
 var errInvalidPodCIDR = errors.New("invalid pod CIDR configuration")
 
-// swarmDefaultAddrPool is the Docker Swarm default overlay address pool.
-const swarmDefaultAddrPool = "10.0.0.0/8"
+// swarmAddrPools returns the overlay address pools that --pod-cidr must not
+// overlap, and whether they describe a swarm that already exists.
+//
+// An existing swarm is authoritative. Its pool is fixed at creation and the
+// InitSwarm phase discards mcr.swarmInstallFlags on such a cluster, so the
+// configured value describes nothing there. Before a swarm exists the configured
+// value is what swarm init will apply, and so is the pool to validate against.
+func (p *ValidateFacts) swarmAddrPools() (pools []string, fromExistingSwarm bool) {
+	if p.Config.Spec.MCR.Metadata != nil && len(p.Config.Spec.MCR.Metadata.SwarmDefaultAddrPool) > 0 {
+		return p.Config.Spec.MCR.Metadata.SwarmDefaultAddrPool, true
+	}
+
+	// docker swarm init accepts --default-addr-pool repeated, so keep every pool.
+	if configured := p.Config.Spec.MCR.SwarmInstallFlags.GetValues("--default-addr-pool"); len(configured) > 0 {
+		return configured, false
+	}
+
+	return []string{swarm.DefaultAddrPoolFallback}, false
+}
+
+// warnSwarmAddrPoolDivergence reports an mcr.swarmInstallFlags --default-addr-pool
+// setting that describes something other than the running cluster. The pool is
+// fixed when the swarm is created, so on an existing cluster the setting is inert
+// and the configuration quietly stops matching the infrastructure.
+//
+// Nothing is reported unless a pool is both explicitly configured and known to
+// differ from the live one, so the common case of a cluster that never set the
+// flag stays silent.
+func (p *ValidateFacts) warnSwarmAddrPoolDivergence() {
+	if p.Config.Spec.MCR.Metadata == nil || len(p.Config.Spec.MCR.Metadata.SwarmDefaultAddrPool) == 0 {
+		return
+	}
+
+	configured := p.Config.Spec.MCR.SwarmInstallFlags.GetValues("--default-addr-pool")
+	if len(configured) == 0 {
+		return
+	}
+
+	live := p.Config.Spec.MCR.Metadata.SwarmDefaultAddrPool
+	if slices.Equal(configured, live) {
+		return
+	}
+
+	log.Warnf(
+		"mcr.swarmInstallFlags sets --default-addr-pool %s but the existing swarm allocates overlay networks from %s. "+
+			"A swarm's address pool is fixed when the swarm is created and cannot be changed on a running cluster, so this "+
+			"setting has no effect here and the cluster no longer matches its configuration. Changing the pool requires "+
+			"dissolving and re-creating the swarm, which destroys every overlay network and service",
+		strings.Join(configured, ","), strings.Join(live, ","),
+	)
+}
 
-// validatePodCIDR checks that --pod-cidr in mke.installFlags does not overlap
-// with the Swarm overlay address pool. Overlapping CIDRs cause the Docker daemon
-// to restart into a broken network state during MKE bootstrap, which silently
-// drops the SSH connection and produces a connection timeout after 20+ minutes.
+// validatePodCIDR checks that --pod-cidr in mke.installFlags does not overlap the
+// Swarm overlay address pool. Overlapping CIDRs cause the Docker daemon to restart
+// into a broken network state during MKE bootstrap, which silently drops the SSH
+// connection and produces a connection timeout after 20+ minutes.
 //
-// If mcr.swarmInstallFlags contains --default-addr-pool, that value is used as
-// the Swarm pool instead of the compiled-in default (10.0.0.0/8).
+// The conflict is fatal only while it is still avoidable, which means before the
+// swarm exists. On an existing swarm the pool cannot be changed, so the conflict
+// is reported and the run continues rather than blocking upgrades of clusters that
+// are already running this way. See PRODENG-3642.
 func (p *ValidateFacts) validatePodCIDR() error {
 	podCIDRStr := p.Config.Spec.MKE.InstallFlags.GetValue("--pod-cidr")
 	if podCIDRStr == "" {
@@ -224,28 +280,48 @@ func (p *ValidateFacts) validatePodCIDR() error {
 		return fmt.Errorf("%w: cannot parse --pod-cidr %q: %w", errInvalidPodCIDR, podCIDRStr, err)
 	}
 
-	// docker swarm init accepts --default-addr-pool repeated, so check every
-	// pool. Fall back to the compiled-in default when none is configured.
-	swarmPools := p.Config.Spec.MCR.SwarmInstallFlags.GetValues("--default-addr-pool")
-	if len(swarmPools) == 0 {
-		swarmPools = []string{swarmDefaultAddrPool}
-	}
+	swarmPools, fromExistingSwarm := p.swarmAddrPools()
+	overlapping := false
 
 	for _, swarmPoolStr := range swarmPools {
 		_, swarmNet, err := net.ParseCIDR(swarmPoolStr)
 		if err != nil {
+			if fromExistingSwarm {
+				// Reported by the daemon rather than written by the user, so
+				// there is nothing in the configuration to correct.
+				log.Warnf("cannot parse the overlay address pool %q reported by the existing swarm: %s", swarmPoolStr, err.Error())
+				continue
+			}
+
 			return fmt.Errorf("%w: cannot parse Swarm address pool %q: %w", errInvalidPodCIDR, swarmPoolStr, err)
 		}
 
-		if swarmNet.Contains(podNet.IP) || podNet.Contains(swarmNet.IP) {
+		if !swarmNet.Contains(podNet.IP) && !podNet.Contains(swarmNet.IP) {
+			continue
+		}
+
+		if !fromExistingSwarm {
 			return fmt.Errorf(
 				"%w: --pod-cidr %s overlaps with the Swarm overlay address pool %s; "+
 					"choose a non-overlapping range or set mcr.swarmInstallFlags --default-addr-pool to a non-conflicting pool",
 				errInvalidPodCIDR, podCIDRStr, swarmPoolStr,
 			)
 		}
+
+		overlapping = true
+
+		log.Warnf(
+			"--pod-cidr %s overlaps the overlay address pool %s of the existing swarm, which can leave the container "+
+				"runtime with a broken network configuration during MKE bootstrap. The pool is fixed when a swarm is "+
+				"created, so mcr.swarmInstallFlags cannot resolve this on a running cluster: either choose a "+
+				"non-overlapping --pod-cidr, or dissolve and re-create the swarm with a non-overlapping pool",
+			podCIDRStr, swarmPoolStr,
+		)
+	}
+
+	if !overlapping {
+		log.Debugf("pod CIDR %s does not overlap with any Swarm pool %v", podCIDRStr, swarmPools)
 	}
 
-	log.Debugf("pod CIDR %s does not overlap with any Swarm pool %v", podCIDRStr, swarmPools)
 	return nil
 }