diff --git a/client/Chart.yaml b/client/Chart.yaml index 47f805e7..aaa6886e 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.42 -appVersion: "1.9.42" +version: 1.9.43 +appVersion: "1.9.43" keywords: - tracebloc - kubernetes diff --git a/client/templates/jobs-manager-pdb.yaml b/client/templates/jobs-manager-pdb.yaml index 3e475064..80b769ad 100644 --- a/client/templates/jobs-manager-pdb.yaml +++ b/client/templates/jobs-manager-pdb.yaml @@ -1,4 +1,39 @@ -{{- if .Values.podDisruptionBudget.jobsManager.create }} +{{- /* + PodDisruptionBudget for the single-replica jobs-manager Deployment. + + `podDisruptionBudget.jobsManager.create` is tri-state: + null (default) -> AUTO, tracking the storage/topology mode: + * multi-node CSI clusters (hostPath.enabled=false, + i.e. AKS/EKS/OpenShift): RENDER. A surprise + autoscaler scale-down or node upgrade could evict + the only pod; minAvailable:1 forces the operator to + drain deliberately. + * single-node / bare-metal installs + (hostPath.enabled=true): SKIP. With one replica on + one node, minAvailable:1 leaves zero allowed voluntary + disruptions, so `kubectl drain`/cordon (node + maintenance, manual k3s upgrade) blocks forever and + the pod has nowhere to reschedule anyway + (tracebloc/client#560). + true -> force render regardless of topology. + false -> never render. +*/}} +{{- $pdb := default dict .Values.podDisruptionBudget.jobsManager -}} +{{- /* Single-node topology gate. `.Values.singleNode` is the dedicated, + unambiguous "this cluster has one schedulable node" signal (tri-state: + null => default to hostPath.enabled; true/false => explicit). It is + deliberately SEPARATE from env.SINGLE_NODE, which only gates the + jobs-manager GPU->CPU fallback and may legitimately be true on a FIXED + MULTI-NODE cluster — using that here would drop eviction protection where + pods can reschedule (client#560 / Bugbot). The installer sets + singleNode=true for node-local single-node k3d (hostPath.enabled=false, + local-path CSI, AGENTS=0); bare-metal single-node keeps the + hostPath.enabled default. */ -}} +{{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} +{{- $singleNode := .Values.singleNode -}} +{{- $isSingleNode := ternary $singleNode $hostPathMode (kindIs "bool" $singleNode) -}} +{{- $create := ternary $pdb.create (not $isSingleNode) (kindIs "bool" $pdb.create) -}} +{{- if $create }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -7,10 +42,11 @@ metadata: labels: {{- include "tracebloc.labels" . | nindent 4 }} spec: - # Single-replica deployment: maxUnavailable: 1 is a no-op (the only pod - # can always go away). Use minAvailable: 1 to actually block voluntary - # disruptions while replicas == 1. Disable when running multi-replica - # jobs-manager managed externally. + # Single-replica deployment: maxUnavailable: 1 would be a no-op (the only pod + # can always go away), so minAvailable: 1 is used to actually block voluntary + # disruptions while replicas == 1. This only renders where a drain has + # somewhere to move the pod (multi-node CSI) or is force-enabled — on + # single-node installs the whole object is skipped (see header above). minAvailable: 1 selector: matchLabels: diff --git a/client/templates/mysql-pdb.yaml b/client/templates/mysql-pdb.yaml index f22e1391..c5c010d1 100644 --- a/client/templates/mysql-pdb.yaml +++ b/client/templates/mysql-pdb.yaml @@ -1,4 +1,31 @@ -{{- if .Values.podDisruptionBudget.mysql.create }} +{{- /* + PodDisruptionBudget for the single-replica mysql Deployment. + + `podDisruptionBudget.mysql.create` is tri-state: + null (default) -> AUTO, tracking the storage/topology mode: + * multi-node CSI clusters (hostPath.enabled=false, + i.e. AKS/EKS/OpenShift): RENDER. Blocks a surprise + autoscaler scale-down / node upgrade from evicting + the only mysql pod without operator intervention. + * single-node / bare-metal installs + (hostPath.enabled=true): SKIP. With one replica on + one node, minAvailable:1 leaves zero allowed voluntary + disruptions, so `kubectl drain`/cordon blocks forever + and the pod (RWO PVC) has nowhere to reschedule + anyway (tracebloc/client#560). + true -> force render regardless of topology. + false -> never render. +*/}} +{{- $pdb := default dict .Values.podDisruptionBudget.mysql -}} +{{- /* Single-node topology gate via the dedicated `.Values.singleNode` signal + (tri-state: null => hostPath.enabled; true/false => explicit), kept SEPARATE + from the GPU-fallback env.SINGLE_NODE which may be true on a fixed multi-node + cluster. See jobs-manager-pdb.yaml for the full rationale (client#560 / Bugbot). */ -}} +{{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} +{{- $singleNode := .Values.singleNode -}} +{{- $isSingleNode := ternary $singleNode $hostPathMode (kindIs "bool" $singleNode) -}} +{{- $create := ternary $pdb.create (not $isSingleNode) (kindIs "bool" $pdb.create) -}} +{{- if $create }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -8,12 +35,13 @@ metadata: {{- include "tracebloc.labels" . | nindent 4 }} app: mysql-client spec: - # Single-replica deployment; minAvailable: 1 blocks *voluntary* disruptions - # (node drains, surge upgrades, cluster-autoscaler scale-downs) while the - # only mysql pod is, well, the only mysql pod. Involuntary kills (kernel - # OOM, node failure) still happen — PriorityClass + memory parity cover - # those. Set podDisruptionBudget.mysql.create=false on multi-replica - # setups managed externally. + # minAvailable: 1 blocks *voluntary* disruptions (node drains, surge + # upgrades, cluster-autoscaler scale-downs) while the only mysql pod is, + # well, the only mysql pod. Involuntary kills (kernel OOM, node failure) + # still happen — PriorityClass + memory parity cover those. This only + # renders where a drain has somewhere to move the pod (multi-node CSI) or + # is force-enabled — on single-node installs the whole object is skipped + # (see header above). minAvailable: 1 selector: matchLabels: diff --git a/client/tests/priority_class_pdb_test.yaml b/client/tests/priority_class_pdb_test.yaml index 33c4b56a..f3916812 100644 --- a/client/tests/priority_class_pdb_test.yaml +++ b/client/tests/priority_class_pdb_test.yaml @@ -106,3 +106,141 @@ tests: value: 1 - notExists: path: spec.maxUnavailable + + # tracebloc/client#560: on a single-node / bare-metal install + # (hostPath.enabled=true) a minAvailable:1 PDB over a 1-replica pod makes the + # sole node undrainable. In AUTO mode (create=null, the default) the PDBs must + # be skipped there so `kubectl drain` can proceed. + - it: should NOT render mysql PDB on single-node/bare-metal (hostPath.enabled=true) in auto mode + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should NOT render jobs-manager PDB on single-node/bare-metal (hostPath.enabled=true) in auto mode + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: true + asserts: + - hasDocuments: + count: 0 + + # AUTO mode on multi-node CSI (hostPath.enabled=false) must still render the + # protective PDBs — this is the default topology and preserves prior behavior. + - it: should render mysql PDB on multi-node CSI (hostPath.enabled=false) in auto mode + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: false + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + - it: should render jobs-manager PDB on multi-node CSI (hostPath.enabled=false) in auto mode + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + # client#560 Bugbot: a node-local single-node install (singleNode=true with + # hostPath.enabled=false — local-path CSI k3d, AGENTS=0) is still single-node, + # so AUTO mode must SKIP the undrainable minAvailable:1 PDB even though + # hostPath is off. The gate uses the dedicated `singleNode` topology flag. + - it: should NOT render mysql PDB on a node-local single-node install (singleNode=true, hostPath.enabled=false) + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: false + singleNode: true + asserts: + - hasDocuments: + count: 0 + - it: should NOT render jobs-manager PDB on a node-local single-node install (singleNode=true, hostPath.enabled=false) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + singleNode: true + asserts: + - hasDocuments: + count: 0 + # Decoupling (client#560 Bugbot): the GPU-fallback env.SINGLE_NODE must NOT + # affect the PDB gate — a fixed MULTI-NODE cluster may set it true, and there + # pods can reschedule, so the PDB must still render to protect against eviction. + - it: should STILL render jobs-manager PDB when env.SINGLE_NODE=true but singleNode is unset (multi-node CSI) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + env: + SINGLE_NODE: "true" + asserts: + - isKind: + of: PodDisruptionBudget + - it: should NOT render jobs-manager PDB when singleNode=false is forced off even on hostPath + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: true + singleNode: false + asserts: + - isKind: + of: PodDisruptionBudget + + # Explicit override: create=true forces the PDB even on single-node/bare-metal + # (for a fixed multi-node bare-metal cluster that still wants drain protection). + - it: create=true should force the mysql PDB even on hostPath (override) + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: true + podDisruptionBudget: + mysql: + create: true + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + - it: create=true should force the jobs-manager PDB even on hostPath (override) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: true + podDisruptionBudget: + jobsManager: + create: true + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + # Explicit override: create=false disables even on multi-node CSI. + - it: create=false should disable the jobs-manager PDB even on CSI (override) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + podDisruptionBudget: + jobsManager: + create: false + asserts: + - hasDocuments: + count: 0 diff --git a/client/values.schema.json b/client/values.schema.json index 386a0390..9c1ee1c9 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -183,6 +183,11 @@ } } }, + "singleNode": { + "type": ["boolean", "null"], + "default": null, + "description": "Cluster topology: does this cluster have a single schedulable node? Gates the AUTO-mode single-replica PodDisruptionBudgets (a minAvailable:1 PDB is undrainable on one node). null => default to hostPath.enabled; true/false => explicit. Deliberately separate from env.SINGLE_NODE (GPU->CPU fallback, which may be true on a fixed multi-node cluster). The installer sets true for node-local single-node k3d (client#560)." + }, "pvc": { "type": "object", "description": "PVC storage sizes", @@ -937,8 +942,9 @@ "type": "object", "properties": { "create": { - "type": "boolean", - "default": true + "type": ["boolean", "null"], + "default": null, + "description": "null (default) = auto: render on multi-node CSI (hostPath.enabled=false), skip on single-node/bare-metal (hostPath.enabled=true). true = force render, false = never render. See tracebloc/client#560." } } }, @@ -946,8 +952,9 @@ "type": "object", "properties": { "create": { - "type": "boolean", - "default": true + "type": ["boolean", "null"], + "default": null, + "description": "null (default) = auto: render on multi-node CSI (hostPath.enabled=false), skip on single-node/bare-metal (hostPath.enabled=true). true = force render, false = never render. See tracebloc/client#560." } } } diff --git a/client/values.yaml b/client/values.yaml index ae5be7fe..f9bb1504 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -128,6 +128,21 @@ hostPath: # appends //data. mysql/logs PV paths are not affected by this key. datasetPath: /tracebloc +# -- Cluster topology: does this cluster have a single schedulable node? +# Gates the AUTO-mode single-replica PodDisruptionBudgets (podDisruptionBudget +# below): a minAvailable:1 PDB on a one-node cluster is undrainable (the pod +# cannot reschedule), so it is skipped when singleNode is effectively true. +# null (default) -> DEFAULT to hostPath.enabled: bare-metal installer clusters +# (hostPath.enabled=true) are single-node; managed dynamic +# clusters (AKS/EKS/OC) are multi-node. +# true / false -> explicit override. +# This is DELIBERATELY separate from env.SINGLE_NODE (which gates the +# jobs-manager GPU->CPU pending-job fallback and may be true on a FIXED +# MULTI-NODE cluster). The installer sets singleNode=true for the node-local +# single-node k3d path (hostPath.enabled=false, local-path CSI, AGENTS=0), where +# hostPath alone would misclassify it as multi-node. +singleNode: null + # -- PVC storage sizes (override per-environment if needed) pvc: mysql: 2Gi @@ -742,15 +757,26 @@ priorityClass: name: "" value: 1000000 -# -- PodDisruptionBudgets. -# mysql is single-replica; minAvailable: 1 blocks voluntary disruptions -# (drains, surge upgrades, autoscaler scale-down) while only one mysql -# pod exists. Disable if you run a multi-replica mysql managed externally. +# -- PodDisruptionBudgets for the single-replica mysql and jobs-manager +# Deployments. `create` is tri-state and defaults to AUTO (null), tracking the +# storage/topology mode: +# null (default) -> multi-node CSI clusters (hostPath.enabled=false: +# AKS/EKS/OpenShift) RENDER a minAvailable:1 PDB, so a +# surprise autoscaler scale-down / node upgrade can't evict +# the only pod without a deliberate operator drain. +# Single-node / bare-metal installs (hostPath.enabled=true) +# SKIP it: with one replica on one node, minAvailable:1 +# allows zero voluntary disruptions, so `kubectl drain` +# (node maintenance, manual k3s upgrade) would block forever +# while the pod has nowhere to reschedule (client#560). +# true -> force render regardless of topology (e.g. a fixed multi-node +# bare-metal cluster that still wants drain protection). +# false -> never render (e.g. a multi-replica workload managed externally). podDisruptionBudget: mysql: - create: true + create: null jobsManager: - create: true + create: null # -- Secrets # These MUST be set via --set, a values file, or an external secret manager. diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index edb41647..023e8924 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -1283,6 +1283,12 @@ storageClass: hostPath: enabled: false + +# node-local is a single schedulable node (common.sh forces AGENTS=0, SERVERS=1), +# so the single-replica PDBs would be undrainable here. hostPath.enabled=false +# would otherwise misclassify this as multi-node, so declare the topology +# explicitly to skip those PDBs and keep the node drainable (client#560). +singleNode: true STORAGE else cat <<'STORAGE' diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 7043c8c2..d7a0fc82 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -8,7 +8,7 @@ b84109f55930b555efc41b0088f9fe8e9741b3eb47e5e993e1fd32d52c5b0ea8 scripts/lib/se 47268af2c406e8b05c57d470469b71ce7358875fe1adda3ecbb566bfb9aed1cc scripts/lib/setup-linux.sh 03c36e012cfb9309f341927eb0c1f8d43f5ca3a7849f3dbfa00cf5b9fdaef62e scripts/lib/cluster.sh 270520b0f6a647da00d653794b0d1334ceffa3e76907af2051101ecb8eae48da scripts/lib/gpu-plugins.sh -22413dc04c8fc49661f626659e3f812e524975c2bec6da58d1fc27459053798e scripts/lib/install-client-helm.sh +a3d1c8a34e7f355d9d5a632c0cbf4b74044827790c02d41f9917d4554dd0cb58 scripts/lib/install-client-helm.sh 61c1c887d158af52d4da4734b3bfa83205b2600ae7a291bfb3074daf3d9ffb55 scripts/lib/install-cli.sh b0bf0a4966461e4257b6765dc4f12877762eee1e72f60ed4b2e322cb6291315a scripts/lib/provision.sh 8bd0deb458e7649723b722d28022018eeeff318068fd7c166756cbdcc3d65806 scripts/lib/assess.sh diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index c72b5e6d..5387a5ac 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -1722,3 +1722,14 @@ _engine_fixture() { [ "$st" -eq 0 ] || { echo "aborted under set -e (st=$st)"; return 1; } [[ "$out" == *REACHED* ]] || { echo "did not reach end: '$out'"; return 1; } } + +@test "install-client-helm declares singleNode: true in the node-local storage branch (client#560)" { + local lib="$BATS_TEST_DIRNAME/../lib/install-client-helm.sh" + # node-local (RFC-0003 Option C, local-path StorageClass) is a single + # schedulable node (AGENTS=0/SERVERS=1), so the values it emits must declare + # singleNode: true — otherwise hostPath.enabled=false misclassifies it as + # multi-node and the chart renders an undrainable minAvailable:1 PDB (#560). + grep -q 'singleNode: true' "$lib" || return 1 + # ...and it must live in the node-local (local-path) values block. + awk '/name: local-path/{f=1} f && /singleNode: true/{found=1} END{exit !found}' "$lib" || return 1 +}