From ba6a306298f7460a4f0a1579a69875289790263a Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Thu, 13 Aug 2026 15:57:21 +0200 Subject: [PATCH 1/4] fix(chart): gate single-replica PDBs on topology so single-node stays drainable (client#560) jobs-manager-pdb.yaml and mysql-pdb.yaml rendered minAvailable:1 unconditionally. For a 1-replica workload the eviction API then permits zero voluntary disruptions, so kubectl drain/cordon blocks forever. On a single-node install that wedges all node maintenance and the pod has nowhere to reschedule anyway. Make podDisruptionBudget.{mysql,jobsManager}.create tri-state: null (default) = AUTO: render on multi-node CSI (hostPath.enabled=false), skip on single-node/bare-metal (hostPath.enabled=true) true = force render false = never render Preserves the protective PDB on multi-node CSI (deliberate operator drain of a single-replica stateful DB) while keeping single-node installs drainable. Bumps Chart.yaml 1.9.39 -> 1.9.40 (chart-version-guard) and adds helm-unittest coverage for both topologies and the overrides. Co-Authored-By: Claude Opus 4.8 --- client/Chart.yaml | 4 +- client/templates/jobs-manager-pdb.yaml | 34 +++++++-- client/templates/mysql-pdb.yaml | 36 +++++++-- client/tests/priority_class_pdb_test.yaml | 93 +++++++++++++++++++++++ client/values.schema.json | 10 ++- client/values.yaml | 23 ++++-- 6 files changed, 176 insertions(+), 24 deletions(-) diff --git a/client/Chart.yaml b/client/Chart.yaml index 2fb97c2c..03c46479 100644 --- a/client/Chart.yaml +++ b/client/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: client description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift type: application -version: 1.9.39 -appVersion: "1.9.39" +version: 1.9.40 +appVersion: "1.9.40" keywords: - tracebloc - kubernetes diff --git a/client/templates/jobs-manager-pdb.yaml b/client/templates/jobs-manager-pdb.yaml index 3e475064..2051e6ef 100644 --- a/client/templates/jobs-manager-pdb.yaml +++ b/client/templates/jobs-manager-pdb.yaml @@ -1,4 +1,27 @@ -{{- if .Values.podDisruptionBudget.jobsManager.create }} +{{- /* + PodDisruptionBudget for the single-replica jobs-manager Deployment. + + `podDisruptionBudget.jobsManager.create` is tri-state: + null (default) -> AUTO, tracking the storage/topology mode: + * multi-node CSI clusters (hostPath.enabled=false, + i.e. AKS/EKS/OpenShift): RENDER. A surprise + autoscaler scale-down or node upgrade could evict + the only pod; minAvailable:1 forces the operator to + drain deliberately. + * single-node / bare-metal installs + (hostPath.enabled=true): SKIP. With one replica on + one node, minAvailable:1 leaves zero allowed voluntary + disruptions, so `kubectl drain`/cordon (node + maintenance, manual k3s upgrade) blocks forever and + the pod has nowhere to reschedule anyway + (tracebloc/client#560). + true -> force render regardless of topology. + false -> never render. +*/}} +{{- $pdb := default dict .Values.podDisruptionBudget.jobsManager -}} +{{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} +{{- $create := ternary $pdb.create (not $hostPathMode) (kindIs "bool" $pdb.create) -}} +{{- if $create }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -7,10 +30,11 @@ metadata: labels: {{- include "tracebloc.labels" . | nindent 4 }} spec: - # Single-replica deployment: maxUnavailable: 1 is a no-op (the only pod - # can always go away). Use minAvailable: 1 to actually block voluntary - # disruptions while replicas == 1. Disable when running multi-replica - # jobs-manager managed externally. + # Single-replica deployment: maxUnavailable: 1 would be a no-op (the only pod + # can always go away), so minAvailable: 1 is used to actually block voluntary + # disruptions while replicas == 1. This only renders where a drain has + # somewhere to move the pod (multi-node CSI) or is force-enabled — on + # single-node installs the whole object is skipped (see header above). minAvailable: 1 selector: matchLabels: diff --git a/client/templates/mysql-pdb.yaml b/client/templates/mysql-pdb.yaml index f22e1391..40c12a40 100644 --- a/client/templates/mysql-pdb.yaml +++ b/client/templates/mysql-pdb.yaml @@ -1,4 +1,25 @@ -{{- if .Values.podDisruptionBudget.mysql.create }} +{{- /* + PodDisruptionBudget for the single-replica mysql Deployment. + + `podDisruptionBudget.mysql.create` is tri-state: + null (default) -> AUTO, tracking the storage/topology mode: + * multi-node CSI clusters (hostPath.enabled=false, + i.e. AKS/EKS/OpenShift): RENDER. Blocks a surprise + autoscaler scale-down / node upgrade from evicting + the only mysql pod without operator intervention. + * single-node / bare-metal installs + (hostPath.enabled=true): SKIP. With one replica on + one node, minAvailable:1 leaves zero allowed voluntary + disruptions, so `kubectl drain`/cordon blocks forever + and the pod (RWO PVC) has nowhere to reschedule + anyway (tracebloc/client#560). + true -> force render regardless of topology. + false -> never render. +*/}} +{{- $pdb := default dict .Values.podDisruptionBudget.mysql -}} +{{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} +{{- $create := ternary $pdb.create (not $hostPathMode) (kindIs "bool" $pdb.create) -}} +{{- if $create }} apiVersion: policy/v1 kind: PodDisruptionBudget metadata: @@ -8,12 +29,13 @@ metadata: {{- include "tracebloc.labels" . | nindent 4 }} app: mysql-client spec: - # Single-replica deployment; minAvailable: 1 blocks *voluntary* disruptions - # (node drains, surge upgrades, cluster-autoscaler scale-downs) while the - # only mysql pod is, well, the only mysql pod. Involuntary kills (kernel - # OOM, node failure) still happen — PriorityClass + memory parity cover - # those. Set podDisruptionBudget.mysql.create=false on multi-replica - # setups managed externally. + # minAvailable: 1 blocks *voluntary* disruptions (node drains, surge + # upgrades, cluster-autoscaler scale-downs) while the only mysql pod is, + # well, the only mysql pod. Involuntary kills (kernel OOM, node failure) + # still happen — PriorityClass + memory parity cover those. This only + # renders where a drain has somewhere to move the pod (multi-node CSI) or + # is force-enabled — on single-node installs the whole object is skipped + # (see header above). minAvailable: 1 selector: matchLabels: diff --git a/client/tests/priority_class_pdb_test.yaml b/client/tests/priority_class_pdb_test.yaml index 33c4b56a..78f898b9 100644 --- a/client/tests/priority_class_pdb_test.yaml +++ b/client/tests/priority_class_pdb_test.yaml @@ -106,3 +106,96 @@ tests: value: 1 - notExists: path: spec.maxUnavailable + + # tracebloc/client#560: on a single-node / bare-metal install + # (hostPath.enabled=true) a minAvailable:1 PDB over a 1-replica pod makes the + # sole node undrainable. In AUTO mode (create=null, the default) the PDBs must + # be skipped there so `kubectl drain` can proceed. + - it: should NOT render mysql PDB on single-node/bare-metal (hostPath.enabled=true) in auto mode + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: true + asserts: + - hasDocuments: + count: 0 + + - it: should NOT render jobs-manager PDB on single-node/bare-metal (hostPath.enabled=true) in auto mode + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: true + asserts: + - hasDocuments: + count: 0 + + # AUTO mode on multi-node CSI (hostPath.enabled=false) must still render the + # protective PDBs — this is the default topology and preserves prior behavior. + - it: should render mysql PDB on multi-node CSI (hostPath.enabled=false) in auto mode + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: false + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + - it: should render jobs-manager PDB on multi-node CSI (hostPath.enabled=false) in auto mode + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + # Explicit override: create=true forces the PDB even on single-node/bare-metal + # (for a fixed multi-node bare-metal cluster that still wants drain protection). + - it: create=true should force the mysql PDB even on hostPath (override) + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: true + podDisruptionBudget: + mysql: + create: true + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + - it: create=true should force the jobs-manager PDB even on hostPath (override) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: true + podDisruptionBudget: + jobsManager: + create: true + asserts: + - isKind: + of: PodDisruptionBudget + - equal: + path: spec.minAvailable + value: 1 + + # Explicit override: create=false disables even on multi-node CSI. + - it: create=false should disable the jobs-manager PDB even on CSI (override) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + podDisruptionBudget: + jobsManager: + create: false + asserts: + - hasDocuments: + count: 0 diff --git a/client/values.schema.json b/client/values.schema.json index 72218ca6..a7b40237 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -894,8 +894,9 @@ "type": "object", "properties": { "create": { - "type": "boolean", - "default": true + "type": ["boolean", "null"], + "default": null, + "description": "null (default) = auto: render on multi-node CSI (hostPath.enabled=false), skip on single-node/bare-metal (hostPath.enabled=true). true = force render, false = never render. See tracebloc/client#560." } } }, @@ -903,8 +904,9 @@ "type": "object", "properties": { "create": { - "type": "boolean", - "default": true + "type": ["boolean", "null"], + "default": null, + "description": "null (default) = auto: render on multi-node CSI (hostPath.enabled=false), skip on single-node/bare-metal (hostPath.enabled=true). true = force render, false = never render. See tracebloc/client#560." } } } diff --git a/client/values.yaml b/client/values.yaml index 76e4644d..aad44e5c 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -681,15 +681,26 @@ priorityClass: name: "" value: 1000000 -# -- PodDisruptionBudgets. -# mysql is single-replica; minAvailable: 1 blocks voluntary disruptions -# (drains, surge upgrades, autoscaler scale-down) while only one mysql -# pod exists. Disable if you run a multi-replica mysql managed externally. +# -- PodDisruptionBudgets for the single-replica mysql and jobs-manager +# Deployments. `create` is tri-state and defaults to AUTO (null), tracking the +# storage/topology mode: +# null (default) -> multi-node CSI clusters (hostPath.enabled=false: +# AKS/EKS/OpenShift) RENDER a minAvailable:1 PDB, so a +# surprise autoscaler scale-down / node upgrade can't evict +# the only pod without a deliberate operator drain. +# Single-node / bare-metal installs (hostPath.enabled=true) +# SKIP it: with one replica on one node, minAvailable:1 +# allows zero voluntary disruptions, so `kubectl drain` +# (node maintenance, manual k3s upgrade) would block forever +# while the pod has nowhere to reschedule (client#560). +# true -> force render regardless of topology (e.g. a fixed multi-node +# bare-metal cluster that still wants drain protection). +# false -> never render (e.g. a multi-replica workload managed externally). podDisruptionBudget: mysql: - create: true + create: null jobsManager: - create: true + create: null # -- Secrets # These MUST be set via --set, a values file, or an external secret manager. From 3e7c699464f9c2e3318f3c77712b6d5a61f5d8c7 Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Fri, 14 Aug 2026 13:15:04 +0200 Subject: [PATCH 2/4] fix(chart): PDB single-node gate honors SINGLE_NODE, not just hostPath (client#560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugbot (Medium): the AUTO-mode PDB skip keyed only on hostPath.enabled, so a node-local single-node install (env.SINGLE_NODE=true, hostPath.enabled=false, local-path CSI k3d, AGENTS=0) still rendered minAvailable:1 — undrainable on a single node, blocking kubectl drain / k3s upgrade. Gate now mirrors jobs-manager-deployment's SINGLE_NODE resolution (explicit env.SINGLE_NODE wins, else hostPath.enabled), so both single-node topologies skip the PDB while multi-node CSI still renders it. Added node-local + SINGLE_NODE=false regression tests. Co-Authored-By: Claude Opus 4.8 --- client/templates/jobs-manager-pdb.yaml | 10 ++++++- client/templates/mysql-pdb.yaml | 8 ++++- client/tests/priority_class_pdb_test.yaml | 36 +++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/client/templates/jobs-manager-pdb.yaml b/client/templates/jobs-manager-pdb.yaml index 2051e6ef..76851d56 100644 --- a/client/templates/jobs-manager-pdb.yaml +++ b/client/templates/jobs-manager-pdb.yaml @@ -19,8 +19,16 @@ false -> never render. */}} {{- $pdb := default dict .Values.podDisruptionBudget.jobsManager -}} +{{- /* Single-node detection mirrors jobs-manager-deployment's SINGLE_NODE + resolution: an explicit env.SINGLE_NODE wins, else default to + hostPath.enabled. A node-local single-node install (SINGLE_NODE=true with + hostPath.enabled=false — local-path CSI k3d, AGENTS=0) must count as + single-node here too, otherwise it renders an undrainable minAvailable:1 + PDB and blocks drain / k3s upgrade (client#560 / Bugbot). */ -}} +{{- $env := default dict .Values.env -}} {{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} -{{- $create := ternary $pdb.create (not $hostPathMode) (kindIs "bool" $pdb.create) -}} +{{- $isSingleNode := ternary (eq (toString (index $env "SINGLE_NODE")) "true") $hostPathMode (hasKey $env "SINGLE_NODE") -}} +{{- $create := ternary $pdb.create (not $isSingleNode) (kindIs "bool" $pdb.create) -}} {{- if $create }} apiVersion: policy/v1 kind: PodDisruptionBudget diff --git a/client/templates/mysql-pdb.yaml b/client/templates/mysql-pdb.yaml index 40c12a40..3620690c 100644 --- a/client/templates/mysql-pdb.yaml +++ b/client/templates/mysql-pdb.yaml @@ -17,8 +17,14 @@ false -> never render. */}} {{- $pdb := default dict .Values.podDisruptionBudget.mysql -}} +{{- /* Single-node detection mirrors jobs-manager-deployment's SINGLE_NODE + resolution (explicit env.SINGLE_NODE wins, else hostPath.enabled) so a + node-local single-node install (SINGLE_NODE=true, hostPath.enabled=false) + also skips the undrainable minAvailable:1 PDB (client#560 / Bugbot). */ -}} +{{- $env := default dict .Values.env -}} {{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} -{{- $create := ternary $pdb.create (not $hostPathMode) (kindIs "bool" $pdb.create) -}} +{{- $isSingleNode := ternary (eq (toString (index $env "SINGLE_NODE")) "true") $hostPathMode (hasKey $env "SINGLE_NODE") -}} +{{- $create := ternary $pdb.create (not $isSingleNode) (kindIs "bool" $pdb.create) -}} {{- if $create }} apiVersion: policy/v1 kind: PodDisruptionBudget diff --git a/client/tests/priority_class_pdb_test.yaml b/client/tests/priority_class_pdb_test.yaml index 78f898b9..bbf32425 100644 --- a/client/tests/priority_class_pdb_test.yaml +++ b/client/tests/priority_class_pdb_test.yaml @@ -155,6 +155,42 @@ tests: path: spec.minAvailable value: 1 + # backend#953-adjacent / client#560 Bugbot: a node-local single-node install + # (env.SINGLE_NODE="true" with hostPath.enabled=false — local-path CSI k3d) + # is still single-node, so AUTO mode must SKIP the undrainable minAvailable:1 + # PDB even though hostPath is off. Gate mirrors the deployment's SINGLE_NODE + # resolution (explicit env.SINGLE_NODE wins over the hostPath default). + - it: should NOT render mysql PDB on a node-local single-node install (SINGLE_NODE=true, hostPath.enabled=false) + template: templates/mysql-pdb.yaml + set: + hostPath: + enabled: false + env: + SINGLE_NODE: "true" + asserts: + - hasDocuments: + count: 0 + - it: should NOT render jobs-manager PDB on a node-local single-node install (SINGLE_NODE=true, hostPath.enabled=false) + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + env: + SINGLE_NODE: "true" + asserts: + - hasDocuments: + count: 0 + - it: should render jobs-manager PDB when SINGLE_NODE=false overrides on multi-node CSI + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: false + env: + SINGLE_NODE: "false" + asserts: + - isKind: + of: PodDisruptionBudget + # Explicit override: create=true forces the PDB even on single-node/bare-metal # (for a fixed multi-node bare-metal cluster that still wants drain protection). - it: create=true should force the mysql PDB even on hostPath (override) From 6a271de2eccfe1ac2de43b63cbcb3464c514b107 Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Fri, 14 Aug 2026 14:57:31 +0200 Subject: [PATCH 3/4] fix(chart): gate PDBs on a dedicated singleNode flag, not env.SINGLE_NODE (client#560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugbot flagged both directions: env.SINGLE_NODE is overloaded — using it to skip PDBs (a) is needed for node-local single-node k3d (hostPath.enabled=false) but (b) wrongly skips on a FIXED MULTI-NODE cluster that sets env.SINGLE_NODE only for the jobs-manager GPU->CPU fallback, dropping eviction protection where pods can reschedule. Introduce a dedicated topology value `singleNode` (tri-state: null => default to hostPath.enabled; true/false => explicit), used ONLY by the AUTO-mode PDB gate and decoupled from env.SINGLE_NODE. Bare-metal single-node keeps the hostPath default; the installer sets singleNode=true for the node-local single-node k3d path (follow-up). Multi-node clusters keep their PDBs regardless of env.SINGLE_NODE. Added node-local, decoupling, and forced-off regression tests + schema. Co-Authored-By: Claude Opus 4.8 --- client/templates/jobs-manager-pdb.yaml | 20 +++++++------ client/templates/mysql-pdb.yaml | 12 ++++---- client/tests/priority_class_pdb_test.yaml | 35 ++++++++++++++--------- client/values.schema.json | 5 ++++ client/values.yaml | 15 ++++++++++ 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/client/templates/jobs-manager-pdb.yaml b/client/templates/jobs-manager-pdb.yaml index 76851d56..80b769ad 100644 --- a/client/templates/jobs-manager-pdb.yaml +++ b/client/templates/jobs-manager-pdb.yaml @@ -19,15 +19,19 @@ false -> never render. */}} {{- $pdb := default dict .Values.podDisruptionBudget.jobsManager -}} -{{- /* Single-node detection mirrors jobs-manager-deployment's SINGLE_NODE - resolution: an explicit env.SINGLE_NODE wins, else default to - hostPath.enabled. A node-local single-node install (SINGLE_NODE=true with - hostPath.enabled=false — local-path CSI k3d, AGENTS=0) must count as - single-node here too, otherwise it renders an undrainable minAvailable:1 - PDB and blocks drain / k3s upgrade (client#560 / Bugbot). */ -}} -{{- $env := default dict .Values.env -}} +{{- /* Single-node topology gate. `.Values.singleNode` is the dedicated, + unambiguous "this cluster has one schedulable node" signal (tri-state: + null => default to hostPath.enabled; true/false => explicit). It is + deliberately SEPARATE from env.SINGLE_NODE, which only gates the + jobs-manager GPU->CPU fallback and may legitimately be true on a FIXED + MULTI-NODE cluster — using that here would drop eviction protection where + pods can reschedule (client#560 / Bugbot). The installer sets + singleNode=true for node-local single-node k3d (hostPath.enabled=false, + local-path CSI, AGENTS=0); bare-metal single-node keeps the + hostPath.enabled default. */ -}} {{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} -{{- $isSingleNode := ternary (eq (toString (index $env "SINGLE_NODE")) "true") $hostPathMode (hasKey $env "SINGLE_NODE") -}} +{{- $singleNode := .Values.singleNode -}} +{{- $isSingleNode := ternary $singleNode $hostPathMode (kindIs "bool" $singleNode) -}} {{- $create := ternary $pdb.create (not $isSingleNode) (kindIs "bool" $pdb.create) -}} {{- if $create }} apiVersion: policy/v1 diff --git a/client/templates/mysql-pdb.yaml b/client/templates/mysql-pdb.yaml index 3620690c..c5c010d1 100644 --- a/client/templates/mysql-pdb.yaml +++ b/client/templates/mysql-pdb.yaml @@ -17,13 +17,13 @@ false -> never render. */}} {{- $pdb := default dict .Values.podDisruptionBudget.mysql -}} -{{- /* Single-node detection mirrors jobs-manager-deployment's SINGLE_NODE - resolution (explicit env.SINGLE_NODE wins, else hostPath.enabled) so a - node-local single-node install (SINGLE_NODE=true, hostPath.enabled=false) - also skips the undrainable minAvailable:1 PDB (client#560 / Bugbot). */ -}} -{{- $env := default dict .Values.env -}} +{{- /* Single-node topology gate via the dedicated `.Values.singleNode` signal + (tri-state: null => hostPath.enabled; true/false => explicit), kept SEPARATE + from the GPU-fallback env.SINGLE_NODE which may be true on a fixed multi-node + cluster. See jobs-manager-pdb.yaml for the full rationale (client#560 / Bugbot). */ -}} {{- $hostPathMode := (default dict .Values.hostPath).enabled | default false -}} -{{- $isSingleNode := ternary (eq (toString (index $env "SINGLE_NODE")) "true") $hostPathMode (hasKey $env "SINGLE_NODE") -}} +{{- $singleNode := .Values.singleNode -}} +{{- $isSingleNode := ternary $singleNode $hostPathMode (kindIs "bool" $singleNode) -}} {{- $create := ternary $pdb.create (not $isSingleNode) (kindIs "bool" $pdb.create) -}} {{- if $create }} apiVersion: policy/v1 diff --git a/client/tests/priority_class_pdb_test.yaml b/client/tests/priority_class_pdb_test.yaml index bbf32425..f3916812 100644 --- a/client/tests/priority_class_pdb_test.yaml +++ b/client/tests/priority_class_pdb_test.yaml @@ -155,38 +155,47 @@ tests: path: spec.minAvailable value: 1 - # backend#953-adjacent / client#560 Bugbot: a node-local single-node install - # (env.SINGLE_NODE="true" with hostPath.enabled=false — local-path CSI k3d) - # is still single-node, so AUTO mode must SKIP the undrainable minAvailable:1 - # PDB even though hostPath is off. Gate mirrors the deployment's SINGLE_NODE - # resolution (explicit env.SINGLE_NODE wins over the hostPath default). - - it: should NOT render mysql PDB on a node-local single-node install (SINGLE_NODE=true, hostPath.enabled=false) + # client#560 Bugbot: a node-local single-node install (singleNode=true with + # hostPath.enabled=false — local-path CSI k3d, AGENTS=0) is still single-node, + # so AUTO mode must SKIP the undrainable minAvailable:1 PDB even though + # hostPath is off. The gate uses the dedicated `singleNode` topology flag. + - it: should NOT render mysql PDB on a node-local single-node install (singleNode=true, hostPath.enabled=false) template: templates/mysql-pdb.yaml set: hostPath: enabled: false - env: - SINGLE_NODE: "true" + singleNode: true asserts: - hasDocuments: count: 0 - - it: should NOT render jobs-manager PDB on a node-local single-node install (SINGLE_NODE=true, hostPath.enabled=false) + - it: should NOT render jobs-manager PDB on a node-local single-node install (singleNode=true, hostPath.enabled=false) template: templates/jobs-manager-pdb.yaml set: hostPath: enabled: false - env: - SINGLE_NODE: "true" + singleNode: true asserts: - hasDocuments: count: 0 - - it: should render jobs-manager PDB when SINGLE_NODE=false overrides on multi-node CSI + # Decoupling (client#560 Bugbot): the GPU-fallback env.SINGLE_NODE must NOT + # affect the PDB gate — a fixed MULTI-NODE cluster may set it true, and there + # pods can reschedule, so the PDB must still render to protect against eviction. + - it: should STILL render jobs-manager PDB when env.SINGLE_NODE=true but singleNode is unset (multi-node CSI) template: templates/jobs-manager-pdb.yaml set: hostPath: enabled: false env: - SINGLE_NODE: "false" + SINGLE_NODE: "true" + asserts: + - isKind: + of: PodDisruptionBudget + - it: should NOT render jobs-manager PDB when singleNode=false is forced off even on hostPath + template: templates/jobs-manager-pdb.yaml + set: + hostPath: + enabled: true + singleNode: false asserts: - isKind: of: PodDisruptionBudget diff --git a/client/values.schema.json b/client/values.schema.json index 7987878f..9c1ee1c9 100644 --- a/client/values.schema.json +++ b/client/values.schema.json @@ -183,6 +183,11 @@ } } }, + "singleNode": { + "type": ["boolean", "null"], + "default": null, + "description": "Cluster topology: does this cluster have a single schedulable node? Gates the AUTO-mode single-replica PodDisruptionBudgets (a minAvailable:1 PDB is undrainable on one node). null => default to hostPath.enabled; true/false => explicit. Deliberately separate from env.SINGLE_NODE (GPU->CPU fallback, which may be true on a fixed multi-node cluster). The installer sets true for node-local single-node k3d (client#560)." + }, "pvc": { "type": "object", "description": "PVC storage sizes", diff --git a/client/values.yaml b/client/values.yaml index 39990b08..f9bb1504 100644 --- a/client/values.yaml +++ b/client/values.yaml @@ -128,6 +128,21 @@ hostPath: # appends //data. mysql/logs PV paths are not affected by this key. datasetPath: /tracebloc +# -- Cluster topology: does this cluster have a single schedulable node? +# Gates the AUTO-mode single-replica PodDisruptionBudgets (podDisruptionBudget +# below): a minAvailable:1 PDB on a one-node cluster is undrainable (the pod +# cannot reschedule), so it is skipped when singleNode is effectively true. +# null (default) -> DEFAULT to hostPath.enabled: bare-metal installer clusters +# (hostPath.enabled=true) are single-node; managed dynamic +# clusters (AKS/EKS/OC) are multi-node. +# true / false -> explicit override. +# This is DELIBERATELY separate from env.SINGLE_NODE (which gates the +# jobs-manager GPU->CPU pending-job fallback and may be true on a FIXED +# MULTI-NODE cluster). The installer sets singleNode=true for the node-local +# single-node k3d path (hostPath.enabled=false, local-path CSI, AGENTS=0), where +# hostPath alone would misclassify it as multi-node. +singleNode: null + # -- PVC storage sizes (override per-environment if needed) pvc: mysql: 2Gi From 49b3b5a89976ae634f1e330f763ba316d373b2d9 Mon Sep 17 00:00:00 2001 From: shujaat hasan Date: Fri, 14 Aug 2026 16:07:06 +0200 Subject: [PATCH 4/4] fix(installer): emit singleNode: true for the node-local storage mode (client#560) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lukas review: the node-local install (TB_STORAGE_MODE=node-local → AGENTS=0, SERVERS=1, hostPath.enabled=false, local-path CSI) is genuinely single-node but nothing emitted the chart's singleNode flag, so it still rendered an undrainable minAvailable:1 PDB — and values.yaml/values.schema.json already claimed (present tense) the installer sets it. Emit singleNode: true in the node-local values branch next to hostPath.enabled: false, making the docs true and closing #560 end-to-end for node-local. Adds a bats test asserting the emit; regenerates scripts/manifest.sha256. Co-Authored-By: Claude Opus 4.8 --- scripts/lib/install-client-helm.sh | 6 ++++++ scripts/manifest.sha256 | 2 +- scripts/tests/install-client-helm.bats | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/scripts/lib/install-client-helm.sh b/scripts/lib/install-client-helm.sh index edb41647..023e8924 100644 --- a/scripts/lib/install-client-helm.sh +++ b/scripts/lib/install-client-helm.sh @@ -1283,6 +1283,12 @@ storageClass: hostPath: enabled: false + +# node-local is a single schedulable node (common.sh forces AGENTS=0, SERVERS=1), +# so the single-replica PDBs would be undrainable here. hostPath.enabled=false +# would otherwise misclassify this as multi-node, so declare the topology +# explicitly to skip those PDBs and keep the node drainable (client#560). +singleNode: true STORAGE else cat <<'STORAGE' diff --git a/scripts/manifest.sha256 b/scripts/manifest.sha256 index 7043c8c2..d7a0fc82 100644 --- a/scripts/manifest.sha256 +++ b/scripts/manifest.sha256 @@ -8,7 +8,7 @@ b84109f55930b555efc41b0088f9fe8e9741b3eb47e5e993e1fd32d52c5b0ea8 scripts/lib/se 47268af2c406e8b05c57d470469b71ce7358875fe1adda3ecbb566bfb9aed1cc scripts/lib/setup-linux.sh 03c36e012cfb9309f341927eb0c1f8d43f5ca3a7849f3dbfa00cf5b9fdaef62e scripts/lib/cluster.sh 270520b0f6a647da00d653794b0d1334ceffa3e76907af2051101ecb8eae48da scripts/lib/gpu-plugins.sh -22413dc04c8fc49661f626659e3f812e524975c2bec6da58d1fc27459053798e scripts/lib/install-client-helm.sh +a3d1c8a34e7f355d9d5a632c0cbf4b74044827790c02d41f9917d4554dd0cb58 scripts/lib/install-client-helm.sh 61c1c887d158af52d4da4734b3bfa83205b2600ae7a291bfb3074daf3d9ffb55 scripts/lib/install-cli.sh b0bf0a4966461e4257b6765dc4f12877762eee1e72f60ed4b2e322cb6291315a scripts/lib/provision.sh 8bd0deb458e7649723b722d28022018eeeff318068fd7c166756cbdcc3d65806 scripts/lib/assess.sh diff --git a/scripts/tests/install-client-helm.bats b/scripts/tests/install-client-helm.bats index c72b5e6d..5387a5ac 100644 --- a/scripts/tests/install-client-helm.bats +++ b/scripts/tests/install-client-helm.bats @@ -1722,3 +1722,14 @@ _engine_fixture() { [ "$st" -eq 0 ] || { echo "aborted under set -e (st=$st)"; return 1; } [[ "$out" == *REACHED* ]] || { echo "did not reach end: '$out'"; return 1; } } + +@test "install-client-helm declares singleNode: true in the node-local storage branch (client#560)" { + local lib="$BATS_TEST_DIRNAME/../lib/install-client-helm.sh" + # node-local (RFC-0003 Option C, local-path StorageClass) is a single + # schedulable node (AGENTS=0/SERVERS=1), so the values it emits must declare + # singleNode: true — otherwise hostPath.enabled=false misclassifies it as + # multi-node and the chart renders an undrainable minAvailable:1 PDB (#560). + grep -q 'singleNode: true' "$lib" || return 1 + # ...and it must live in the node-local (local-path) values block. + awk '/name: local-path/{f=1} f && /singleNode: true/{found=1} END{exit !found}' "$lib" || return 1 +}