From 43c4e8c5b96c2570da88cab6f386d41d0119db67 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 7 Jul 2026 17:33:25 -0400 Subject: [PATCH 1/6] controller: set pool Degraded/NodeDegraded when nodes report errors When a daemon reports Degraded=True on a BootcNode, the controller now sets the pool's Degraded condition with reason NodeDegraded and a message listing the affected node names. Assisted-by: Pi (Claude Opus 4.6) Signed-off-by: Jonathan Lebon --- internal/controller/rollout.go | 28 +++++++ internal/controller/rollout_envtest_test.go | 91 +++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/internal/controller/rollout.go b/internal/controller/rollout.go index 6e41a7c..f6eab90 100644 --- a/internal/controller/rollout.go +++ b/internal/controller/rollout.go @@ -59,6 +59,11 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv rs := buildRolloutState(log, ownedBootcNodes) + // Flag degraded nodes at the pool level. NodeConflict (set during + // membership sync) takes priority, so we only set NodeDegraded if + // the pool isn't already degraded for another reason. + syncNodeDegradedCondition(pool, rs.degraded) + // Free reboot slots for nodes that have successfully rebooted into // the desired image. This runs before computing available slots so // that freed capacity is immediately usable for new candidates. @@ -358,6 +363,29 @@ func buildRolloutState(log logr.Logger, ownedBootcNodes map[string]*bootcv1alpha return rs } +// syncNodeDegradedCondition sets Degraded/NodeDegraded on the pool if any +// nodes are degraded. It respects priority: if the pool is already degraded +// for another reason (e.g. NodeConflict), it does not overwrite it. +// XXX(jl): Or... should this be a new condition type so it can be surfaced in +// parallel? Let's see how this approach feels and iterate. +func syncNodeDegradedCondition(pool *bootcv1alpha1.BootcNodePool, degraded []*bootcv1alpha1.BootcNode) { + if len(degraded) == 0 { + return + } + if apimeta.IsStatusConditionTrue(pool.Status.Conditions, bootcv1alpha1.PoolDegraded) { + return + } + names := nodeNames(degraded) + // Sort so the message is stable across reconciles. + slices.Sort(names) + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.PoolDegraded, + Status: metav1.ConditionTrue, + Reason: bootcv1alpha1.PoolNodeDegraded, + Message: fmt.Sprintf("Degraded nodes: %s", strings.Join(names, ", ")), + }) +} + // resolveMaxUnavailable computes the effective maxUnavailable value from the // pool's rollout spec. Defaults to 1 when unset. A value of 0 is allowed and // means no reboot slots are available (effectively paused). Returns an diff --git a/internal/controller/rollout_envtest_test.go b/internal/controller/rollout_envtest_test.go index 17b53f6..d71fe64 100644 --- a/internal/controller/rollout_envtest_test.go +++ b/internal/controller/rollout_envtest_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" @@ -130,6 +131,71 @@ func TestSimpleRollout(t *testing.T) { } } +// TestDegradedNodeSetsPoolCondition verifies that when a daemon reports +// Degraded=True on a BootcNode, the pool is marked Degraded/NodeDegraded, +// and the rollout continues on non-degraded nodes. +func TestDegradedNodeSetsPoolCondition(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + const poolName = "degraded-pool" + + // Create 2 worker nodes. + nodeNames := []string{"degraded-w1", "degraded-w2"} + for _, name := range nodeNames { + name := name + node := testutil.NewK8sNode(name, testutil.WorkerLabels()) + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, node) + }) + } + + // Create pool targeting digest B with maxUnavailable: 1. + pool := testutil.NewPool(poolName, testImageDigestRefB, + testutil.WithWorkerSelector(), + testutil.WithMaxUnavailable(intstr.FromInt32(1)), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, pool) + }) + + // Wait for BootcNodes to be created. + for _, name := range nodeNames { + name := name + g.Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: name}, &bootcv1alpha1.BootcNode{}) + }).Should(Succeed()) + } + + // Simulate w1 as degraded (staging failed) and w2 as staged. + simulateDaemonDegraded(g, ctx, "degraded-w1", testDigestA) + simulateDaemonStatus(g, ctx, "degraded-w2", testDigestA, bootcv1alpha1.NodeReasonStaged) + + // Verify pool is Degraded/NodeDegraded mentioning w1. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKey{Name: poolName}, &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolNodeDegraded), + HaveField("Message", ContainSubstring("degraded-w1")), + ))) + + // Verify rollout continues on the non-degraded node: w2 should get + // a reboot slot despite w1 being degraded. + g.Eventually(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: "degraded-w2"}, &bn) + return bn.Annotations, err + }).Should(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), "non-degraded node should get a reboot slot") +} + // simulateDaemonStatus writes BootcNode status as if the daemon had // reported the given booted digest and Idle condition reason. func simulateDaemonStatus(g Gomega, ctx context.Context, nodeName, bootedDigest, idleReason string) { @@ -157,6 +223,31 @@ func simulateDaemonStatus(g Gomega, ctx context.Context, nodeName, bootedDigest, g.Expect(k8sClient.Status().Update(ctx, &bn)).To(Succeed()) } +// simulateDaemonDegraded writes BootcNode status as if the daemon had +// reported the given booted digest with Degraded=True (e.g. staging failed). +func simulateDaemonDegraded(g Gomega, ctx context.Context, nodeName, bootedDigest string) { + var bn bootcv1alpha1.BootcNode + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &bn)).To(Succeed()) + + bn.Status.Booted = &bootcv1alpha1.ImageInfo{ + Image: "quay.io/example/myos@" + bootedDigest, + ImageDigest: bootedDigest, + } + apimeta.SetStatusCondition(&bn.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.NodeIdle, + Status: metav1.ConditionFalse, + Reason: bootcv1alpha1.NodeReasonStaging, + }) + apimeta.SetStatusCondition(&bn.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.NodeDegraded, + Status: metav1.ConditionTrue, + Reason: bootcv1alpha1.NodeReasonError, + Message: "simulated staging failure", + }) + + g.Expect(k8sClient.Status().Update(ctx, &bn)).To(Succeed()) +} + // setNodeReady sets the Ready condition on a K8s Node to True. In // envtest there is no kubelet, so this simulates the node becoming // healthy after a reboot. From 77d1f2e66a0b73693f2c37bc3c59315d0b5d303a Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 8 Jul 2026 11:58:48 -0400 Subject: [PATCH 2/6] api: add PoolRolloutHalted degraded reason Add a new Degraded condition reason for when the controller stops assigning reboot slots because 2 or more nodes in reboot slots are unhealthy (Degraded or not Ready after rebooting into the target image). No behavioral changes yet; the constant is wired up in a follow-up commit. Assisted-by: Pi (Claude Opus 4.6) Signed-off-by: Jonathan Lebon --- api/v1alpha1/bootcnodepool_types.go | 5 +++++ docs/ARCHITECTURE.md | 19 ++++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/bootcnodepool_types.go b/api/v1alpha1/bootcnodepool_types.go index 1818307..66991df 100644 --- a/api/v1alpha1/bootcnodepool_types.go +++ b/api/v1alpha1/bootcnodepool_types.go @@ -40,6 +40,11 @@ const ( // affected nodes and their issues. PoolNodeDegraded string = "NodeDegraded" + // PoolRolloutHalted means the controller stopped assigning new + // reboot slots because 2 or more nodes in reboot slots are + // unhealthy (Degraded or not Ready). + PoolRolloutHalted string = "RolloutHalted" + // PoolInvalidSpec means the pool's spec contains invalid values that // the controller cannot process (e.g. an image ref that is not a // digest ref, or a malformed nodeSelector). diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index bd5c3b3..2a6271b 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -130,15 +130,16 @@ status: Pool conditions and their reasons: -| Condition | Status | Reason | Meaning | -|-----------|--------|-------------------|---------------------------------------------------| -| UpToDate | True | AllUpdated | All nodes are running targetDigest | -| UpToDate | False | RolloutInProgress | Nodes are actively being updated | -| UpToDate | False | Paused | Updates pending but pool is paused | -| Degraded | True | NodeConflict | Node selector overlaps with another pool | -| Degraded | True | NodeDegraded | At least one node has errors or isn't converging | -| Degraded | True | InvalidSpec | Pool spec contains invalid values | -| Degraded | False | Healthy | No issues | +| Condition | Status | Reason | Meaning | +|-----------|--------|-------------------|-----------------------------------------------------| +| UpToDate | True | AllUpdated | All nodes are running targetDigest | +| UpToDate | False | RolloutInProgress | Nodes are actively being updated | +| UpToDate | False | Paused | Updates pending but pool is paused | +| Degraded | True | NodeConflict | Node selector overlaps with another pool | +| Degraded | True | NodeDegraded | At least one node has errors or isn't converging | +| Degraded | True | RolloutHalted | 2+ unhealthy nodes in reboot slots; rollout stopped | +| Degraded | True | InvalidSpec | Pool spec contains invalid values | +| Degraded | False | Healthy | No issues | The `UpToDate` condition is determined by the controller by comparing `spec.desiredImage` vs `status.booted.imageDigest` across all nodes in the pool. From 222da9af7d48125583f083139892714e485db330 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 8 Jul 2026 12:01:54 -0400 Subject: [PATCH 3/6] controller: halt rollout when 2+ nodes in reboot slots are unhealthy When 2 or more nodes occupying reboot slots are unhealthy (BootcNode Degraded after booting the target digest, or K8s Node not Ready after update), the controller stops assigning new reboot slots and sets the pool Degraded with reason RolloutHalted. A single unhealthy node might be a hardware issue, but two suggest a bad image. The rollout resumes automatically once the unhealthy count drops below the threshold (e.g. nodes recover or are manually fixed). Assisted-by: Pi (Claude Opus 4.6) Signed-off-by: Jonathan Lebon --- internal/controller/rollout.go | 90 +++++++++++++- internal/controller/rollout_envtest_test.go | 131 ++++++++++++++++++++ 2 files changed, 218 insertions(+), 3 deletions(-) diff --git a/internal/controller/rollout.go b/internal/controller/rollout.go index f6eab90..a7090b3 100644 --- a/internal/controller/rollout.go +++ b/internal/controller/rollout.go @@ -24,6 +24,15 @@ import ( bootcv1alpha1 "github.com/bootc-dev/bootc-operator/api/v1alpha1" ) +// Hardcode to 2 for now; might make this configurable later, or dynamically +// adjusted. The rationale is that 1 unhealthy node might be a one-off. But 2 +// unhealthy nodes becomes a pattern that might indicate a bad image. Now, this +// _probably_ should scale with number of nodes somehow. E.g. a 500 node +// cluster could tolerate more unhealthy nodes. Just starting conservative here +// and we can adjust. See +// https://github.com/bootc-dev/bootc-operator/issues/99. +const unhealthySlotHaltThreshold = 2 + // rolloutState holds the classified BootcNodes for a single reconcile // pass. type rolloutState struct { @@ -71,13 +80,34 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv return fmt.Errorf("freeing completed slots: %w", err) } + // Check for unhealthy nodes on the target digest in reboot slots. If + // 2+ are unhealthy, halt the rollout. Note that freeCompletedSlots() + // already removed Ready nodes, so any upToDate node still in a slot is + // implied to be NotReady. + unhealthy := findUnhealthySlots(rs, pool.Status.TargetDigest) + + rolloutHalted := len(unhealthy) >= unhealthySlotHaltThreshold + if rolloutHalted { + syncRolloutHaltedCondition(pool, unhealthy) + log.Info("Rollout halted: 2+ unhealthy nodes in reboot slots", + "unhealthyInSlots", len(unhealthy)) + + // Return early to block new slot assignments and new/restarted + // drains. Note that collectDrainResults() already ran above, + // so drains that were in-flight and completed between + // reconciles will still have their results collected and + // desiredImageState set to Booted. Trying to "un-drain" and + // uncordon fully drained nodes is out of scope for now. + return nil + } + maxUnavail, err := resolveMaxUnavailable(pool, rs.nodeCount()) if err != nil { return err } - avail := max(0, maxUnavail-rs.occupiedSlots) - candidates := selectDrainCandidates(rs.staged, avail) + availableSlots := max(0, maxUnavail-rs.occupiedSlots) + candidates := selectDrainCandidates(rs.staged, availableSlots) log.V(1).Info("Rollout state", "upToDate", len(rs.upToDate), @@ -89,7 +119,7 @@ func (r *BootcNodePoolReconciler) driveRollout(ctx context.Context, pool *bootcv "unclassified", nodeNames(rs.unclassified), "occupiedSlots", rs.occupiedSlots, "maxUnavailable", maxUnavail, - "availableSlots", avail, + "availableSlots", availableSlots, "candidates", nodeNames(candidates), ) @@ -386,6 +416,60 @@ func syncNodeDegradedCondition(pool *bootcv1alpha1.BootcNodePool, degraded []*bo }) } +// unhealthySlot identifies a node in a reboot slot that is unhealthy. +type unhealthySlot struct { + name string + reason string // "Degraded" or "NotReady" +} + +// findUnhealthySlots returns nodes occupying reboot slots that are unhealthy. +// Two categories qualify: +// - Degraded nodes (marked by the daemon itself) that booted the target +// digest (the image itself may be bad). +// - UpToDate nodes still holding a slot after freeCompletedSlots (not +// Ready). I.e. the daemon doesn't see anything wrong, but there's something +// wrong preventing the kubelet from working correctly. +func findUnhealthySlots(rs *rolloutState, targetDigest string) []unhealthySlot { + var result []unhealthySlot + for _, bn := range rs.degraded { + if !metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { + continue + } + // Note we count nil Booted as unhealthy here; this really + // shouldn't happen because clearly the daemon came up at least + // once in this node's history to be able to get to a reboot + // slot. And so Booted should always be set here. + if bn.Status.Booted == nil || bn.Status.Booted.ImageDigest == targetDigest { + result = append(result, unhealthySlot{name: bn.Name, reason: "Degraded"}) + } + } + for _, bn := range rs.upToDate { + if metav1.HasAnnotation(bn.ObjectMeta, bootcv1alpha1.AnnotationInRebootSlot) { + // Still has annotation after freeCompletedSlots → not Ready. + result = append(result, unhealthySlot{name: bn.Name, reason: "NotReady"}) + } + } + return result +} + +// syncRolloutHaltedCondition sets Degraded/RolloutHalted on the pool. It +// implicitly takes priority over NodeDegraded (which checks for existing +// daemon-driven degraded status) by running later than +// syncNodeDegradedCondition. +func syncRolloutHaltedCondition(pool *bootcv1alpha1.BootcNodePool, unhealthy []unhealthySlot) { + details := make([]string, len(unhealthy)) + for i, u := range unhealthy { + details[i] = u.name + ": " + u.reason + } + slices.Sort(details) + apimeta.SetStatusCondition(&pool.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.PoolDegraded, + Status: metav1.ConditionTrue, + Reason: bootcv1alpha1.PoolRolloutHalted, + Message: fmt.Sprintf("Rollout halted: 2+ unhealthy nodes in reboot slots (%s)", strings.Join(details, ", ")), + }) +} + // resolveMaxUnavailable computes the effective maxUnavailable value from the // pool's rollout spec. Defaults to 1 when unset. A value of 0 is allowed and // means no reboot slots are available (effectively paused). Returns an diff --git a/internal/controller/rollout_envtest_test.go b/internal/controller/rollout_envtest_test.go index d71fe64..1e41ca9 100644 --- a/internal/controller/rollout_envtest_test.go +++ b/internal/controller/rollout_envtest_test.go @@ -196,6 +196,137 @@ func TestDegradedNodeSetsPoolCondition(t *testing.T) { }).Should(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), "non-degraded node should get a reboot slot") } +// TestUnhealthyNodesHaltRollout verifies that when 2+ nodes in reboot slots +// are unhealthy, the controller stops assigning new slots and sets +// Degraded/RolloutHalted on the pool. It also verifies recovery: when +// unhealthy nodes are fixed, the rollout resumes. +func TestUnhealthyNodesHaltRollout(t *testing.T) { + g := NewWithT(t) + g.SetDefaultEventuallyTimeout(pollTimeout) + g.SetDefaultEventuallyPollingInterval(pollInterval) + ctx := context.Background() + + const poolName = "halt-pool" + + // Create 4 worker nodes. + nodeNames := []string{"halt-w1", "halt-w2", "halt-w3", "halt-w4"} + for _, name := range nodeNames { + name := name + node := testutil.NewK8sNode(name, testutil.WorkerLabels()) + g.Expect(k8sClient.Create(ctx, node)).To(Succeed()) + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, node) + }) + } + + // Create pool targeting digest B with maxUnavailable: 3. + pool := testutil.NewPool(poolName, testImageDigestRefB, + testutil.WithWorkerSelector(), + testutil.WithMaxUnavailable(intstr.FromInt32(3)), + ) + g.Expect(k8sClient.Create(ctx, pool)).To(Succeed()) + t.Cleanup(func() { + _ = k8sClient.Delete(ctx, pool) + }) + + // Wait for BootcNodes to be created. + for _, name := range nodeNames { + name := name + g.Eventually(func() error { + return k8sClient.Get(ctx, client.ObjectKey{Name: name}, &bootcv1alpha1.BootcNode{}) + }).Should(Succeed()) + } + + // Simulate all 4 nodes as Staged (booted old image, staged new one). + for _, name := range nodeNames { + simulateDaemonStatus(g, ctx, name, testDigestA, bootcv1alpha1.NodeReasonStaged) + } + + // With maxUnavailable: 3, the first 3 nodes (alphabetical) should get + // reboot slots. Wait for w1, w2, w3 to get slots. + for _, name := range nodeNames[:3] { + name := name + g.Eventually(func(g Gomega) { + var bn bootcv1alpha1.BootcNode + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: name}, &bn)).To(Succeed()) + g.Expect(bn.Annotations).To(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), + "node %s should have reboot slot", name) + g.Expect(bn.Spec.DesiredImageState).To(Equal(bootcv1alpha1.DesiredImageStateBooted), + "node %s should have desiredImageState Booted", name) + }).Should(Succeed()) + } + + // w4 should not have a slot (all 3 slots are occupied). + var bn4 bootcv1alpha1.BootcNode + g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: "halt-w4"}, &bn4)).To(Succeed()) + g.Expect(bn4.Annotations).NotTo(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), + "w4 should not have a slot yet") + + // Simulate w1 as degraded (booted the target digest and Ready but daemon says it's bad). + simulateDaemonDegraded(g, ctx, "halt-w1", testDigestB) + setNodeReady(g, ctx, "halt-w1") + + // Simulate w2 as upToDate but not Ready (booted target, node not Ready). + simulateDaemonStatus(g, ctx, "halt-w2", testDigestB, bootcv1alpha1.NodeReasonIdle) + // Note: we do NOT call setNodeReady for w2, so it stays not Ready. + + // Simulate w3 as having successfully completed (booted target, Idle, Ready). + simulateDaemonStatus(g, ctx, "halt-w3", testDigestB, bootcv1alpha1.NodeReasonIdle) + setNodeReady(g, ctx, "halt-w3") + + // Wait for pool to be Degraded/RolloutHalted. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKey{Name: poolName}, &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionTrue), + HaveField("Reason", bootcv1alpha1.PoolRolloutHalted), + HaveField("Message", ContainSubstring("halt-w1")), + HaveField("Message", ContainSubstring("halt-w2")), + ))) + + // w3's slot should be freed (it's healthy and Ready), but w4 should + // still not get a slot because the rollout is halted. + g.Eventually(func() (map[string]string, error) { + var bn3 bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: "halt-w3"}, &bn3) + return bn3.Annotations, err + }).ShouldNot(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), "w3 slot should be freed") + + g.Consistently(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: "halt-w4"}, &bn) + return bn.Annotations, err + }, "2s", pollInterval).ShouldNot(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), "w4 should not get a slot while rollout is halted") + + // Fix w1: clear degraded, report as booted on target and Idle. + simulateDaemonStatus(g, ctx, "halt-w1", testDigestB, bootcv1alpha1.NodeReasonIdle) + setNodeReady(g, ctx, "halt-w1") + + // Fix w2: set node Ready. + setNodeReady(g, ctx, "halt-w2") + + // Rollout should resume: w4 should now get a slot. + g.Eventually(func() (map[string]string, error) { + var bn bootcv1alpha1.BootcNode + err := k8sClient.Get(ctx, client.ObjectKey{Name: "halt-w4"}, &bn) + return bn.Annotations, err + }).Should(HaveKey(bootcv1alpha1.AnnotationInRebootSlot), "w4 should get a slot after rollout resumes") + + // Pool should no longer be Degraded/RolloutHalted. + g.Eventually(func() ([]metav1.Condition, error) { + var p bootcv1alpha1.BootcNodePool + err := k8sClient.Get(ctx, client.ObjectKey{Name: poolName}, &p) + return p.Status.Conditions, err + }).Should(ContainElement(And( + HaveField("Type", bootcv1alpha1.PoolDegraded), + HaveField("Status", metav1.ConditionFalse), + HaveField("Reason", bootcv1alpha1.PoolHealthy), + ))) +} + // simulateDaemonStatus writes BootcNode status as if the daemon had // reported the given booted digest and Idle condition reason. func simulateDaemonStatus(g Gomega, ctx context.Context, nodeName, bootedDigest, idleReason string) { From 8509b70020d44b7bd604442a2c006532532e78be Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Wed, 8 Jul 2026 13:04:55 -0400 Subject: [PATCH 4/6] test: use status patch in setNodeReady to avoid conflict errors The controller concurrently patches spec.unschedulable on the same Node, which bumps the resourceVersion. A Status().Update fails in that case because it checks the top-level resourceVersion. Switch to Status().Patch which avoids the conflict entirely. Assisted-by: Pi (Claude Opus 4.6) Signed-off-by: Jonathan Lebon --- internal/controller/rollout_envtest_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/controller/rollout_envtest_test.go b/internal/controller/rollout_envtest_test.go index 1e41ca9..fcd7ce1 100644 --- a/internal/controller/rollout_envtest_test.go +++ b/internal/controller/rollout_envtest_test.go @@ -386,19 +386,21 @@ func setNodeReady(g Gomega, ctx context.Context, nodeName string) { var node corev1.Node g.Expect(k8sClient.Get(ctx, client.ObjectKey{Name: nodeName}, &node)).To(Succeed()) + modified := node.DeepCopy() + // Replace any existing Ready condition, preserving other conditions. var filtered []corev1.NodeCondition - for _, c := range node.Status.Conditions { + for _, c := range modified.Status.Conditions { if c.Type != corev1.NodeReady { filtered = append(filtered, c) } } - node.Status.Conditions = append(filtered, corev1.NodeCondition{ + modified.Status.Conditions = append(filtered, corev1.NodeCondition{ Type: corev1.NodeReady, Status: corev1.ConditionTrue, LastHeartbeatTime: metav1.Now(), LastTransitionTime: metav1.Now(), }) - g.Expect(k8sClient.Status().Update(ctx, &node)).To(Succeed()) + g.Expect(k8sClient.Status().Patch(ctx, modified, client.MergeFrom(&node))).To(Succeed()) } From 3f5a9a0352016d0187e7dc8026d5673a6712a3da Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 14 Jul 2026 22:40:22 -0400 Subject: [PATCH 5/6] envtest/rollout: use SetStatusCondition when simulating daemon To match the other simulation code in that file and also because it seems weird to wipe out the `NodeDegraded` type condition type entirely. Signed-off-by: Jonathan Lebon --- internal/controller/rollout_envtest_test.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/controller/rollout_envtest_test.go b/internal/controller/rollout_envtest_test.go index fcd7ce1..a9c3b2b 100644 --- a/internal/controller/rollout_envtest_test.go +++ b/internal/controller/rollout_envtest_test.go @@ -342,14 +342,17 @@ func simulateDaemonStatus(g Gomega, ctx context.Context, nodeName, bootedDigest, if idleReason == bootcv1alpha1.NodeReasonIdle { idleStatus = metav1.ConditionTrue } - bn.Status.Conditions = []metav1.Condition{ - { - Type: bootcv1alpha1.NodeIdle, - Status: idleStatus, - Reason: idleReason, - LastTransitionTime: metav1.Now(), - }, - } + apimeta.SetStatusCondition(&bn.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.NodeIdle, + Status: idleStatus, + Reason: idleReason, + }) + // Clear Degraded when simulating a healthy status. + apimeta.SetStatusCondition(&bn.Status.Conditions, metav1.Condition{ + Type: bootcv1alpha1.NodeDegraded, + Status: metav1.ConditionFalse, + Reason: bootcv1alpha1.NodeReasonHealthy, + }) g.Expect(k8sClient.Status().Update(ctx, &bn)).To(Succeed()) } From dbaa36ed1872e5ed076a4c5918d090e855868c11 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Tue, 14 Jul 2026 23:05:24 -0400 Subject: [PATCH 6/6] docs/IMPLEMENTATION_PLAN: check off fsnotify This was done in 21c3169 ("daemon: add StatusWatcher for fsnotify + polling"). Signed-off-by: Jonathan Lebon --- docs/IMPLEMENTATION_PLAN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 2dc165c..79c78ef 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -240,7 +240,7 @@ and verify the daemon stages it and reports `Idle=False reason=Staged`. Then set `desiredImageState: Booted`, verify the node reboots into the new image and the daemon reports `Idle=True`. -### 4d. fsnotify + polling +### 4d. fsnotify + polling ✅ - fsnotify watch on `/proc/1/root/ostree/bootc` for CHMOD events - Fallback: also try `/proc/1/root/sysroot/state/deploy` (for composefs