feat: CR controller checks for host overload scenario and resolves via reservation re-placements

cmd/manager/main.go

@@ -668,6 +668,20 @@ func main() {
 				os.Exit(1)
 			}
 		}
+
+		if commitmentsConfig.ReservationController.EnableOversubscriptionCheck {
+			reservationControllerMonitor := commitments.NewReservationControllerMonitor()
+			metrics.Registry.MustRegister(&reservationControllerMonitor)
+
+			if err := (&commitments.HostOversubscriptionController{
+				Client:  multiclusterClient,
+				Conf:    commitmentsConfig.ReservationController,
+				Monitor: &reservationControllerMonitor,
+			}).SetupWithManager(mgr, multiclusterClient); err != nil {
+				setupLog.Error(err, "unable to create controller", "controller", "HostOversubscription")
+				os.Exit(1)
+			}
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "datasource-controllers") {
 		setupLog.Info("enabling controller", "controller", "datasource-controllers")

helm/bundles/cortex-nova/templates/alerts.yaml

@@ -749,4 +749,74 @@ spec:
           resource router is mapping the same object to multiple clusters, or an
           object was created out-of-band on the wrong cluster. Investigate the
           affected resources and the routing configuration.
+
+    {{- if .Values.kvm.enabled }}
+    - alert: CortexNovaHostReservationsOversubscribed
+      # Fires when the sum of running VM allocations + reservation blocks (committed +
+      # failover) exceeds the host's effective capacity for CPU or memory.
+      # This can happen due to: concurrent slot creation with stale informer cache,
+      # operator-driven VM migrations where the slot stays on the old host, or
+      # capacity changes (e.g. hardware replacement changing EffectiveCapacity).
+      # The 10m hold-off tolerates the known migration window: after a VM departs,
+      # the slot remains on the old host until the usage reconciler cleans it up.
+      # Note: `reserved` only counts Ready reservations — violations during the
+      # initial unready window (slot just created) are not captured by this alert.
+      expr: |
+        (
+          max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="utilized"})
+          + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="reserved"})
+          + max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_usage{type="failover"})
+          - max by (compute_host, availability_zone, resource) (cortex_kvm_host_capacity_total)
+        ) > 0
+      for: 15m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "Host {{ "{{" }} $labels.compute_host {{ "}}" }} reservation blocks exceed capacity for {{ "{{" }} $labels.resource {{ "}}" }}"
+        description: >
+          The total of running VM allocations and reservation blocks (committed resource +
+          failover) on host {{ "{{" }} $labels.compute_host {{ "}}" }} exceeds its effective
+          capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}.
+          This means the host is over-subscribed and committed resource or failover guarantees may not be
+          satisfied. Common causes: VM migrations with slot not yet reclaimed, outdated/stale reservations, or out of sync issues. 
+          If problem remains, inspect the reservations on this host and check the CR/failover controller logs.
+    - alert: CortexCommittedResourceHostOversubscribed
+      expr: max by (host, az, resource) (cortex_committed_resource_host_oversubscribed) > 0
+      for: 15m
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "Host {{ "{{" }} $labels.host {{ "}}" }} CR slots exceed effective capacity for {{ "{{" }} $labels.resource {{ "}}" }}"
+        description: >
+          Committed resource reservation slots on host {{ "{{" }} $labels.host {{ "}}" }} exceed
+          its effective capacity for {{ "{{" }} $labels.resource {{ "}}" }} by {{ "{{" }} $value | humanize {{ "}}" }}.
+          The oversubscription controller was unable to resolve this by evicting unallocated slots.
+          Inspect the CR reservations on this host and check the oversubscription controller logs.
+    - alert: CortexCommittedResourceHostOversubscriptionEvictionsHigh
+      expr: sum by (az) (increase(cortex_committed_resource_host_oversubscribed_unplaced_reservations_total{service="cortex-nova-metrics"}[1h])) > 10
+      labels:
+        context: committed-resource-capacity
+        dashboard: cortex-status-dashboard/cortex-status-dashboard
+        service: cortex
+        severity: warning
+        support_group: workload-management
+        playbook: docs/support/playbook/cortex/alerts/committed-resource-capacity
+      annotations:
+        summary: "High CR slot eviction rate in AZ {{ "{{" }} $labels.az {{ "}}" }}"
+        description: >
+          More than 10 committed resource reservation slots were evicted due to oversubscription
+          in AZ {{ "{{" }} $labels.az {{ "}}" }} within the last hour ({{ "{{" }} $value | humanize {{ "}}" }} evictions).
+          This indicates recurring oversubscription events — investigate capacity changes,
+          concurrent slot creation races, or VM migration patterns in this AZ.
+    {{- end }}
 {{- end }}

helm/bundles/cortex-nova/values.yaml

@@ -185,6 +185,16 @@ cortex-scheduling-controllers:
       # How long after a VM is allocated to a reservation before it is expected to appear
       # on the target host; allocations not confirmed within this window are removed
       allocationGracePeriod: "15m"
+      # How long to wait after detecting host over-subscription before evicting reservation slots. 
+      # Gives other controllers (e.g. failover) time to self-heal.
+      oversubscriptionGracePeriod: "3m"
+      # Minimum time between consecutive over-subscription checks for the same host.
+      # Should be shorter than oversubscriptionGracePeriod.
+      oversubscriptionMinCheckInterval: "1m"
+      # Enable host over-subscription detection
+      enableOversubscriptionCheck: true
+      # When false, violations are detected and exposed via metrics but no slots are evicted.
+      enableOversubscriptionUnplaceReservations: true
       # URL of the nova external scheduler API for placement decisions
       schedulerURL: "http://localhost:8080/scheduler/nova/external"
       # Keystone credentials used to resolve domain IDs to domain names for the

internal/scheduling/lib/filter_monitor_test.go

@@ -33,6 +33,7 @@ func TestMonitorFilter(t *testing.T) {
 	fm := monitorFilter(mockFilter, "test-filter", monitor)
 	if fm == nil {
 		t.Fatal("expected filter monitor, got nil")
+		return
 	}
 	if fm.filter == nil {
 		t.Error("expected filter to be set")
@@ -109,6 +110,7 @@ func TestFilterMonitor_Run(t *testing.T) {
 	}
 	if result == nil {
 		t.Fatal("expected result, got nil")
+		return
 	}
 	if len(result.Activations) != 2 {
 		t.Errorf("expected 2 activations, got %d", len(result.Activations))

internal/scheduling/lib/filter_validation_test.go

@@ -20,6 +20,7 @@ func TestValidateFilter(t *testing.T) {
 
 	if validator == nil {
 		t.Fatal("expected validator but got nil")
+		return
 	}
 	if validator.Filter != filter {
 		t.Error("expected filter to be set in validator")

internal/scheduling/lib/filter_weigher_pipeline_step_test.go

@@ -121,6 +121,7 @@ func TestBaseFilterWeigherPipelineStep_IncludeAllHostsFromRequest(t *testing.T)
 
 			if result == nil {
 				t.Fatal("expected result but got nil")
+				return
 			}
 			if len(result.Activations) != tt.expectedCount {
 				t.Errorf("expected %d activations, got %d", tt.expectedCount, len(result.Activations))

internal/scheduling/lib/weigher_monitor_test.go

@@ -33,6 +33,7 @@ func TestMonitorWeigher(t *testing.T) {
 	wm := monitorWeigher(mockWeigher, "test-weigher", monitor)
 	if wm == nil {
 		t.Fatal("expected weigher monitor, got nil")
+		return
 	}
 	if wm.weigher == nil {
 		t.Error("expected weigher to be set")
@@ -109,6 +110,7 @@ func TestWeigherMonitor_Run(t *testing.T) {
 	}
 	if result == nil {
 		t.Fatal("expected result, got nil")
+		return
 	}
 	if len(result.Activations) != 2 {
 		t.Errorf("expected 2 activations, got %d", len(result.Activations))

internal/scheduling/lib/weigher_validation_test.go

@@ -21,6 +21,7 @@ func TestValidateWeigher(t *testing.T) {
 
 	if validator == nil {
 		t.Fatal("expected validator but got nil")
+		return
 	}
 	if validator.Weigher != weigher {
 		t.Error("expected weigher to be set in validator")

internal/scheduling/nova/filter_weigher_pipeline_controller_test.go

@@ -479,6 +479,7 @@ func TestFilterWeigherPipelineController_ProcessNewDecisionFromAPI(t *testing.T)
 				ready := meta.FindStatusCondition(history.Status.Conditions, v1alpha1.HistoryConditionReady)
 				if ready == nil {
 					t.Fatalf("expected Ready condition to be set")
+					return
 				}
 				if ready.Status != metav1.ConditionTrue {
 					t.Errorf("expected Ready condition status True, got %s", ready.Status)

internal/scheduling/pods/plugins/filters/filter_node_available_test.go

@@ -318,6 +318,7 @@ func TestNodeAvailableFilter_Run(t *testing.T) {
 
 			if result == nil {
 				t.Fatal("expected result to be non-nil")
+				return
 			}
 
 			if len(result.Activations) != len(tt.expected) {

internal/scheduling/reservations/capacity_accounting.go

@@ -25,6 +25,7 @@ import (
 // res itself is excluded from step 3 to avoid subtracting its own block from free capacity.
 // Returns false when the hypervisor has no capacity data.
 func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv1.Hypervisor, res *v1alpha1.Reservation) bool {
+	// TODO consider refactor with HostFreeCapacity
 	effCap := hv.Status.EffectiveCapacity
 	if effCap == nil {
 		effCap = hv.Status.Capacity
@@ -75,6 +76,49 @@ func HostHasCapacityForReservation(allReservations []v1alpha1.Reservation, hv hv
 	return true
 }
 
+// HostFreeCapacity computes the remaining free capacity on hv after subtracting
+// hv.Status.Allocation and UnusedReservationCapacity for all reservations on this host.
+// Negative values indicate over-subscription for that resource.
+// Returns nil when the hypervisor has no capacity data.
+// Reservations not targeting this host (via Spec.TargetHost or Status.Host) are ignored.
+func HostFreeCapacity(hostReservations []v1alpha1.Reservation, hv hv1.Hypervisor) map[hv1.ResourceName]resource.Quantity {
+	// TODO consider refactor with HostHasCapacityForReservation
+	effCap := hv.Status.EffectiveCapacity
+	if effCap == nil {
+		effCap = hv.Status.Capacity
+	}
+	if effCap == nil {
+		return nil
+	}
+
+	free := make(map[hv1.ResourceName]resource.Quantity, len(effCap))
+	for rn, qty := range effCap {
+		free[rn] = qty.DeepCopy()
+	}
+	for rn, allocated := range hv.Status.Allocation {
+		if f, ok := free[rn]; ok {
+			f.Sub(allocated)
+			free[rn] = f
+		}
+	}
+	for i := range hostReservations {
+		res := &hostReservations[i]
+		if res.Spec.TargetHost == "" {
+			continue // evicted/unplaced; status.Host may lag until reconcile
+		}
+		if res.Spec.TargetHost != hv.Name && res.Status.Host != hv.Name {
+			continue
+		}
+		for rn, block := range UnusedReservationCapacity(res, false) {
+			if f, ok := free[rn]; ok {
+				f.Sub(block)
+				free[rn] = f
+			}
+		}
+	}
+	return free
+}
+
 // UnusedReservationCapacity returns the resources a Reservation should block on its host(s).
 // This is the single source of truth used by both the capacity controller and
 // filter_has_enough_capacity to ensure consistent accounting.