diff --git a/.github/workflows/chart.yml b/.github/workflows/chart.yml index 2c8bac6..8ecb64a 100644 --- a/.github/workflows/chart.yml +++ b/.github/workflows/chart.yml @@ -99,6 +99,41 @@ jobs: --set mode=enforce >/dev/null 2>&1; then echo "mode: enforce with empty enforceScope was not refused"; exit 1 fi + - name: "Assert node-containment RBAC (ADR-0040 §6) is posture-derived on enforceRung: node" + run: | + # The metadata-only nodes READ (fleet observation for the rails) is ALWAYS ON, + # even under mode: audit (the default) — a read is harmless, and ContainNode + # stays structurally unarmed either way (ADR-0040 §5). + audit=$(helm template protector charts/protector --namespace protector) + echo "$audit" | grep -A3 'resources: ."nodes".' | grep -q 'get' \ + || { echo "audit default is missing the always-on nodes read grant"; exit 1; } + # But NO install below the top rung — including audit and enforce+quarantine — + # may ever grant the nodes WRITE (patch). + if echo "$audit" | grep -A3 'resources: ."nodes".' | grep -q 'patch'; then + echo "audit default leaked the nodes patch (write) grant"; exit 1 + fi + quarantine=$(helm template protector charts/protector --namespace protector \ + --set mode=enforce --set 'enforceScope.namespaces={payments}' \ + --set enforceRung=quarantine) + if echo "$quarantine" | grep -A3 'resources: ."nodes".' | grep -q 'patch'; then + echo "enforce+quarantine leaked the nodes patch (write) grant"; exit 1 + fi + # enforce + enforceRung: node is the ONLY posture that grants nodes patch — + # cluster-scoped (no per-namespace Role/RoleBinding exists for it, unlike + # NetworkPolicy) — alongside its predecessor rungs' NetworkPolicy write grant. + node=$(helm template protector charts/protector --namespace protector \ + --set mode=enforce --set 'enforceScope.namespaces={payments}' \ + --set enforceRung=node) + echo "$node" | grep -A3 'resources: ."nodes".' | grep -q 'patch' \ + || { echo "enforce+node did not derive the nodes patch (write) grant"; exit 1; } + echo "$node" | grep -A3 'resources: ."networkpolicies".' | grep -q 'create' \ + || { echo "enforce+node lost the predecessor NetworkPolicy write grant"; exit 1; } + # enforceRung: node without mode: enforce must be refused at render time too + # (defense-in-depth alongside the engine's own startup refusal). + if helm template protector charts/protector --namespace protector \ + --set enforceRung=node >/dev/null 2>&1; then + echo "enforceRung: node under mode: audit was not refused"; exit 1 + fi - name: "Assert namespace-scoped enforce tightens NetworkPolicy write to per-namespace Roles" run: | # NetworkPolicy is namespaced and the engine only writes cuts inside diff --git a/charts/protector/README.md b/charts/protector/README.md index d217def..c123c41 100644 --- a/charts/protector/README.md +++ b/charts/protector/README.md @@ -111,6 +111,26 @@ rung: A rung implies its narrower predecessor — `quarantine` still arms the edge-cut too — so this is one ordered position to reason about, not independent per-cut toggles. +**A third, explicit opt-in — `enforceRung: node` — makes `ContainNode` ELIGIBLE TO +PROPOSE, protector's first NODE-write RBAC (ADR-0040).** Strictly above `quarantine` +(`edge-cut < quarantine < node`): a proven pod-boundary break (a host-credential read, +a root escalation paired with a container-escape primitive, a kernel tamper, or +co-resident model-confirmed compromise) escalates from a pod-scoped cut to a *proposal* +to cordon the `Host` node plus a default-deny sweep of its other labelled pods. +**`ContainNode` is PROPOSE-ONLY and NEVER auto-applied, at any rung** — ADR-0040 §5 +makes a node cut propose-first *by construction* (a real node always has alive +collateral): arming this rung only makes a rails-clean cut surface as an actionable +proposal (the deterministic rails — never the control plane, at most one node +concurrently, a two-schedulable-worker floor, ownership-gated revert — are pre-checked +and named on it) instead of a bare "not auto-enabled" line; an actual cordon requires an +explicit human action outside protector. Requires `mode: enforce` with a non-empty +`enforceScope`, exactly like every other rung; the chart and the engine both refuse to +start otherwise: + +```sh +--set enforceRung=node +``` + **The fail-closed webhook and the actuation RBAC are derived from the same `enforceScope`** — they can no longer drift from what the gates enforce. By default the audit webhook **fails open** (`failurePolicy: Ignore`, so a protector outage never @@ -231,11 +251,17 @@ The engine's reversible network cut is armed together with the webhook surfaces `mode: enforce` (above) — there is no separate engine arming switch — but *how much* of it is armed is its own ORDERED ladder, `enforceRung` (ADR-0035): `edge-cut` (the default) arms only the surgical edge-cut; `quarantine` also arms the broader -quarantines. In `mode: audit` the engine is always dry-run, regardless of `enforceRung`; -under `mode: enforce` it applies the rung's armed cut(s) on a corroborated attack path -whose endpoints are within `enforceScope`, and the NetworkPolicy write grant is derived +quarantines; `node` (ADR-0040 §6) additionally makes `ContainNode`, protector's first +node-write action, ELIGIBLE TO PROPOSE — never to auto-apply, at any rung (ADR-0040 §5's +propose-first-by-construction rule), gated the whole way down by its own deterministic +rails. In `mode: audit` the engine is always dry-run, regardless of `enforceRung`; under +`mode: enforce` it applies the rung's armed cut(s) on a corroborated attack path whose +endpoints are within `enforceScope`, and the NetworkPolicy write grant is derived from `mode` (they arm together, independent of the rung — the RBAC grant covers the same -object kind at either rung). Choose the CNI mechanism the cut renders with: +object kind at either rung). The `nodes` `patch` grant, unlike NetworkPolicy's, is +derived from `enforceRung: node` specifically (see "Enforce" above) — it is the one +grant on this ladder that does NOT arm together with the others. Choose the CNI +mechanism the cut renders with: ```sh --set engine.actuator=networkpolicy # default — any NetworkPolicy-enforcing CNI (ADR-0010) @@ -275,7 +301,7 @@ Requires the `protector-agent` image and probes load-tested on your kernel (see | `mode` | `audit` | **The posture switch** (ADR-0021). `enforce` arms the webhooks + the engine's cut up to `enforceRung`, all in `enforceScope`. | | `enforceScope.namespaces` | `[]` | Namespace names to enforce (used only under `mode: enforce`). No wildcard. | | `enforceScope.labels` | `{}` | Pod labels (`key: value`) to enforce anywhere; labels behave like namespaces. | -| `enforceRung` | `edge-cut` | **The engine's cut-severity ladder** (ADR-0035, used only under `mode: enforce`). `edge-cut` arms only the surgical `DenyNetworkPath` cut; `quarantine` also arms the broader entry/workload quarantines. Ordered — `quarantine` implies `edge-cut`. | +| `enforceRung` | `edge-cut` | **The engine's cut-severity ladder** (ADR-0035, used only under `mode: enforce`). `edge-cut` arms only the surgical `DenyNetworkPath` cut; `quarantine` also arms the broader entry/workload quarantines; `node` (ADR-0040 §6) makes `ContainNode` (cordon + co-resident deny) ELIGIBLE TO PROPOSE — never to auto-apply, at any rung — and grants the cluster-scoped `nodes` `patch` RBAC. Ordered — each rung implies its predecessor(s). `node` requires `mode: enforce` with a non-empty `enforceScope`. | | `image.tag` | `""` → chart `appVersion` | Pin a cosign-signed semver tag. | | `imagePullSecrets` | `[]` | protector publishes to a public ghcr repo. | | `engine.enabled` | `true` | The mitigation engine (the product). | diff --git a/charts/protector/templates/clusterrole.yaml b/charts/protector/templates/clusterrole.yaml index 8d3e89c..335f0d4 100644 --- a/charts/protector/templates/clusterrole.yaml +++ b/charts/protector/templates/clusterrole.yaml @@ -49,6 +49,17 @@ rules: - apiGroups: ["policy.linkerd.io"] resources: ["servers", "authorizationpolicies", "meshtlsauthentications"] verbs: ["get", "list", "watch"] + # Node fleet observation (ADR-0040 §3/§6) — metadata-only (name, the + # node-role.kubernetes.io/control-plane label, spec.unschedulable, protector's own + # cordon-ownership annotation; never .data, there is none on a Node). ALWAYS ON, + # like every other read grant above: this is a READ, harmless under mode: audit (the + # ContainNode class stays structurally unarmed either way, ADR-0040 §5), and the + # node-containment rails need real fleet data the moment `enforceRung: node` is armed + # — gating the read behind the rung too would leave the very first pass after arming + # blind. The WRITE this read feeds (below) stays posture-derived. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] {{- if eq .Values.mode "enforce" }} # mode: enforce — the network actuator creates/deletes its own (managed-by: protector) # additive, reversible objects to quarantine/sever a corroborated attack path. The @@ -80,5 +91,23 @@ rules: resources: ["adminnetworkpolicies"] verbs: ["create", "delete", "patch"] {{- end }} + {{- if eq .Values.enforceRung "node" }} + # mode: enforce + enforceRung: node (ADR-0040 §6, ADR-0021 posture-derived RBAC) — + # the top rung's ONLY new write surface: Node.spec.unschedulable (cordon/uncordon). + # ContainNode is PROPOSE-ONLY and never auto-applied at any rung (ADR-0040 §5's + # propose-first-by-construction rule) — this grant exists for the standard ledger + # self-revert/break-glass UNCORDON path (a standing cut, however it became standing, + # must still be liftable) and for a future human-approval-to-apply flow, gated the + # whole way down by the SAME deterministic rails regardless of this grant existing + # (respond::actuator::node_containment::cordon_decision). `nodes` is IRREDUCIBLY + # CLUSTER-SCOPED — unlike NetworkPolicy there is no namespaced Node kind to narrow + # this to — so the grant is cluster-wide even for a namespace-only enforceScope; + # `enforceScope`/the arming ladder still confine WHICH mitigations may be proposed, + # this only grants the mechanism. `patch` only: the engine never creates or deletes + # a Node. + - apiGroups: [""] + resources: ["nodes"] + verbs: ["patch"] + {{- end }} {{- end }} {{- end }} diff --git a/charts/protector/templates/deployment.yaml b/charts/protector/templates/deployment.yaml index a51cf1a..78cd362 100644 --- a/charts/protector/templates/deployment.yaml +++ b/charts/protector/templates/deployment.yaml @@ -9,8 +9,14 @@ {{- end }} {{- /* ADR-0035 arming-ladder validation (defense-in-depth; the engine also falls back to the narrowest rung on an unrecognized value): enforceRung must be a rung on the ladder. */}} -{{- if not (or (eq .Values.enforceRung "edge-cut") (eq .Values.enforceRung "quarantine")) }} -{{- fail (printf "enforceRung must be 'edge-cut' or 'quarantine', got %q" .Values.enforceRung) }} +{{- if not (or (eq .Values.enforceRung "edge-cut") (eq .Values.enforceRung "quarantine") (eq .Values.enforceRung "node")) }} +{{- fail (printf "enforceRung must be 'edge-cut', 'quarantine', or 'node', got %q" .Values.enforceRung) }} +{{- end }} +{{- /* ADR-0040 §6 (defense-in-depth; the engine also refuses at startup): the top rung is + protector's first node-write action class — never render it without mode: enforce (which + the check above already requires to carry a non-empty enforceScope). */}} +{{- if and (eq .Values.enforceRung "node") (ne .Values.mode "enforce") }} +{{- fail "enforceRung: node requires mode: enforce with a non-empty enforceScope — node containment (ADR-0040) is protector's first node-write action and must never be configured without both enforcement gates armed" }} {{- end }} {{- /* Feed-fetcher sidecar wiring. When feedSync is ON (the default), a native @@ -243,11 +249,13 @@ spec: value: {{ join "," .Values.enforceScope.namespaces | quote }} - name: PROTECTOR_ENFORCE_SCOPE_LABELS value: {{ include "protector.enforceScopeLabels" . | quote }} - # The ordered network-cut arming ladder (ADR-0035): how far `mode: enforce` + # The ordered cut-severity arming ladder (ADR-0035): how far `mode: enforce` # is armed. `edge-cut` (the default) arms ONLY the surgical DenyNetworkPath # cut; `quarantine` is an explicit second opt-in that adds the broader - # QuarantineEntry/QuarantineWorkload denies (implying the edge-cut rung too - # — one ordered position, not independent toggles). Ignored under mode: audit. + # QuarantineEntry/QuarantineWorkload denies (implying the edge-cut rung too); + # `node` is an explicit third opt-in that additionally arms ContainNode, the + # first node-write action class (ADR-0040 §6) — one ordered position, not + # independent toggles. Ignored under mode: audit. - name: PROTECTOR_ENFORCE_RUNG value: {{ .Values.enforceRung | quote }} {{- if .Values.rekor.enabled }} diff --git a/charts/protector/values.yaml b/charts/protector/values.yaml index 910348c..c940785 100644 --- a/charts/protector/values.yaml +++ b/charts/protector/values.yaml @@ -66,7 +66,7 @@ enforceScope: namespaces: [] labels: {} -# The ORDERED network-cut arming ladder (ADR-0035) — how far `mode: enforce` is armed. +# The ORDERED cut-severity arming ladder (ADR-0035) — how far `mode: enforce` is armed. # One position, not independent per-cut toggles: a higher rung always implies its # narrower predecessor. Used only when mode: enforce; ignored under mode: audit. # edge-cut (DEFAULT) — arms ONLY the surgical DenyNetworkPath edge-cut, the @@ -75,6 +75,18 @@ enforceScope: # entry quarantine and the compromised-workload quarantine (both still the # same additive/reversible network-deny mechanism, ADR-0010). Escalate here # only after baking confidence on edge-cut. +# node — an explicit THIRD opt-in, strictly above quarantine (ADR-0040 §6): makes +# ContainNode (cordon the Host a proven pod-boundary break is scheduled on, +# plus a co-resident default-deny sweep) ELIGIBLE to be PROPOSED — protector's +# first node-write RBAC. ContainNode is PROPOSE-ONLY: ADR-0040 §5 makes a node +# cut propose-first BY CONSTRUCTION (a real node always has alive collateral), +# so it is NEVER auto-applied at any rung — the deterministic rails (never the +# control plane, at most one node, a two-schedulable-worker floor, +# ownership-gated revert) are a bound on the proposal, not a replacement for +# human review; an actual cordon needs an explicit human action outside +# protector. Requires mode: enforce with a non-empty enforceScope, exactly +# like every other rung — the engine refuses to start otherwise. Escalate here +# only after baking confidence on quarantine. enforceRung: edge-cut image: diff --git a/engine/src/engine/break_glass_node_tests.rs b/engine/src/engine/break_glass_node_tests.rs index 84a5c18..d7f26ef 100644 --- a/engine/src/engine/break_glass_node_tests.rs +++ b/engine/src/engine/break_glass_node_tests.rs @@ -1,30 +1,36 @@ -//! Engine-level acceptance tests for ADR-0040 §5's `ContainNode` revert wiring: break-glass -//! (ADR-0036) and the standard ledger self-revert lifecycle (ADR-0017) must uncordon a -//! standing node-containment cut within one pass — never leave it silently in place through -//! the wrong-shaped generic network `actuator`. Split out of `break_glass_tests.rs` purely to -//! keep every file under the 1,000-line cap (repo CLAUDE.md); `use super::*` resolves to the -//! engine module, matching every sibling `*_tests.rs` file. +//! Engine-level acceptance tests for ADR-0040 §5's `ContainNode` revert wiring AND its +//! propose-only invariant: break-glass (ADR-0036) and the standard ledger self-revert +//! lifecycle (ADR-0017) must uncordon a standing node-containment cut within one pass — +//! never leave it silently in place through the wrong-shaped generic network `actuator` — +//! while `ContainNode` itself must NEVER be reachable through any apply call, at any +//! arming/scope/rails state (there is no such call site to reach: `ContainNode` is +//! propose-first by construction, ADR-0040 §5). Split out of `break_glass_tests.rs` purely +//! to keep every file under the 1,000-line cap (repo CLAUDE.md); `use super::*` resolves to +//! the engine module, matching every sibling `*_tests.rs` file. //! -//! Deliberately does NOT depend on the `node` arming rung (ADR-0040 §6, a separate ticket) or -//! on any node-observation adapter (`respond::actuator::node_containment`'s own doc): +//! Deliberately does NOT depend on the `node` arming rung +//! (`respond::actuator::arming_ladder::ArmingRung::Node`) or a live fleet watch: //! `EnabledActions::enable(ProposedAction::ContainNode)` arms the class directly, bypassing -//! `EnabledActions::from_names` (which has no `node` name yet), and each test seeds its own -//! `NodeFact`/actuator double via `Engine::with_node_fact`/`Engine::with_node_containment_actuator` -//! rather than a live fleet watch. The PROPOSAL half is real production machinery — the model -//! naming a boundary-broken workload, the menu escalating it to `ContainNode` +//! `EnabledActions::from_names` (which has no `node` name), and each test seeds its own +//! `NodeFact`/actuator double via `Engine::with_node_fact`/`Engine::with_node_containment_actuator`. +//! The PROPOSAL half is real production machinery — the model naming a boundary-broken +//! workload, the menu escalating it to `ContainNode` //! (`reason::proof::boundary_break`, `reason::adjudicate::incident::menu`), the ledger -//! tracking it — only the "this was already applied" step is synthesized directly into the -//! action log, standing in for the sibling ticket's future apply-side rail wiring. +//! tracking it, and (for the revert tests below) `Engine::process`'s own proposal-side gate +//! (`respond::actuator::node_containment::evaluate_proposal`) all run for real. Only the +//! "this was already applied" step in the REVERT tests is synthesized directly into the +//! action log — standing in for a HYPOTHETICAL future human-approval-to-apply flow +//! (out of scope of every ticket to date); there is no code path that does this for real. use super::*; use crate::engine::graph::attack::AttackRef; use crate::engine::graph::{NodeKey, SecurityGraph}; use crate::engine::observe::Snapshot; use crate::engine::reason::adjudicate::incident::{Assessment, IncidentDecision, Menu}; -use crate::engine::respond::actuator::{Actuation, node_containment}; +use crate::engine::respond::actuator::{Actuation, Actuator, node_containment}; use serde_json::json; -use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; /// A unique temp flag path for a test, without a temp-file crate — mirrors /// `break_glass_tests::temp_flag_path`. @@ -421,3 +427,83 @@ async fn clearing_break_glass_leaves_the_node_classs_posture_byte_identical() { assert_eq!(after_clear.action, baseline.action); assert_eq!(engaged_then_cleared.actions.active_count(), 0); } + +/// A spy [`Actuator`] that records every `apply()` call it receives, by action, via a +/// shared handle — the acceptance-level proof that `ContainNode` has NO reachable apply +/// call through the GENERIC actuator, at any arming/scope/rails state. Mirrors +/// [`RecordingNodeContainmentActuator`]'s pattern of sharing state through a clone rather +/// than downcasting the `Box` `Engine` owns. +struct SpyActuator { + applied: Arc>>, +} + +#[async_trait::async_trait] +impl Actuator for SpyActuator { + async fn apply(&self, mitigation: &Mitigation) -> Actuation { + self.applied.lock().unwrap().push(mitigation.action); + Actuation::DryRun + } + async fn revert(&self, _mitigation: &Mitigation) -> Actuation { + Actuation::DryRun + } +} + +/// ADR-0040 §5 acceptance proof: `ContainNode` never reaches ANY apply call — not the +/// generic network [`Actuator::apply`] (already structurally forbidden by `decide()`'s +/// `is_additive_live()` check, `respond::actuator::tests::decide_forbids_contain_node_even_when_the_class_would_be_enabled`), +/// and there is no SEPARATE node-apply seam in `Engine` to call instead (only +/// [`Engine::with_node_containment_actuator`] for REVERT exists — there is no +/// `with_node_containment_actuator`-for-apply counterpart). Armed, unscoped (so +/// `enforceScope` can't be the reason nothing applies), and every deterministic rail +/// passing (three plain workers, no control-plane, no one-node-cap conflict) — the +/// maximally-eligible case for the proposal gate to say "yes" — still applies nothing. +#[tokio::test] +async fn contain_node_never_reaches_an_apply_call_even_fully_eligible() { + let applied = Arc::new(Mutex::new(Vec::new())); + let actuator = SpyActuator { + applied: applied.clone(), + }; + let mut engine = Engine::new( + EnabledActions::from_names(["judgement"]).enable(ProposedAction::ContainNode), + ActuationScope::unscoped(), + Box::new(actuator), + Box::new(NamesWhateverTheMenuResolves), + ) + .with_node_fact(node_containment::NodeFact { + name: "node-1".to_string(), + control_plane: false, + schedulable: true, + owned_by_protector: false, + }) + .with_node_fact(node_containment::NodeFact { + name: "node-2".to_string(), + control_plane: false, + schedulable: true, + owned_by_protector: false, + }) + .with_node_fact(node_containment::NodeFact { + name: "node-3".to_string(), + control_plane: false, + schedulable: true, + owned_by_protector: false, + }); + let snap = boundary_broken_snapshot(true); + + // Proves eligibility was genuinely reached (not vacuously "nothing to apply because + // nothing was even proposed"): the real proof + menu resolution produced a + // `ContainNode` mitigation over a healthy 3-node fleet with no control-plane/ + // one-node-cap conflict — exactly the fleet shape `evaluate_proposal`'s own unit + // tests confirm resolves to `ProposalOutcome::Proposed`. + let mitigation = contain_node_mitigation(&mut engine, &snap).await; + assert_eq!(mitigation.action, ProposedAction::ContainNode); + + assert!( + applied.lock().unwrap().is_empty(), + "no apply call of any kind must ever be made for a ContainNode-eligible pass" + ); + assert_eq!( + engine.actions.active_count(), + 0, + "ContainNode must never be recorded as an applied/standing action" + ); +} diff --git a/engine/src/engine/metrics.rs b/engine/src/engine/metrics.rs index 7a40e00..36f1b76 100644 --- a/engine/src/engine/metrics.rs +++ b/engine/src/engine/metrics.rs @@ -106,14 +106,25 @@ pub(super) struct EngineMetrics { /// operator can page on "this fired at all" rather than poll the gauge. pub(super) break_glass_transitions: opentelemetry::metrics::Counter, /// `ProposedAction::ContainNode` (ADR-0040) events, by `event` - /// (`proposed`/`applied`/`reverted`/`rail_refused`) and, for a `rail_refused` event, - /// `reason` (`control-plane`/`one-node-cap`/`worker-floor`/`unlabelled`/`not-owned` — - /// `respond::actuator::node_containment::RailRefusal::metric_reason`). A separate - /// counter from the generic [`Self::mitigations`] one: `ContainNode` is - /// `is_additive_live() == false`, so it never reaches `mitigations`' `applied`/ - /// `reverted` labels through the generic auto-apply path — and its deterministic rails - /// must be observable even in shadow (no `node` arming rung wired yet), so a refusal is - /// counted regardless of whether anything is armed. + /// (`proposed`/`reverted`/`rail_refused`) and, for a `rail_refused` event, `reason` + /// (`control-plane`/`one-node-cap`/`worker-floor`/`unlabelled`/`not-owned`/ + /// `unknown-node` — `respond::actuator::node_containment::RailRefusal::metric_reason`). + /// **There is no `applied` event and never will be**: ADR-0040 §5 makes `ContainNode` + /// propose-first BY CONSTRUCTION (never auto-applied, at any arming rung), so this + /// counter only ever observes proposal/refusal/revert, never an apply. `proposed` + /// fires from TWO call sites at different cadences, both meaning "there is a live, + /// reviewable node-containment candidate right now": edge-triggered the moment the + /// ledger first proposes the mitigation at all (`Engine::process`'s ledger-delta + /// loop, unconditional — fires even under `mode: audit`), and level-triggered every + /// pass thereafter while it is ALSO armed (`node` rung), in `enforceScope`, and every + /// deterministic rail passes (`respond::actuator::node_containment::evaluate_proposal`) + /// — the stronger claim "this is currently a rails-clean, actionable proposal", not + /// just "boundary_break fired once". A separate counter from the generic + /// [`Self::mitigations`] one: `ContainNode` never reaches `mitigations`' `applied`/ + /// `reverted` labels through the generic auto-apply path (it is + /// `is_additive_live() == false`), and its deterministic rails must be observable + /// even when unarmed (a rail refusal is exactly as meaningful in shadow), so a + /// refusal is counted regardless of whether anything is armed. pub(super) contain_node: opentelemetry::metrics::Counter, } @@ -239,8 +250,9 @@ impl EngineMetrics { contain_node: m .u64_counter("protector.engine.contain_node") .with_description( - "ContainNode (ADR-0040) events by event (proposed/applied/reverted/\ - rail_refused) and, for a refusal, reason.", + "ContainNode (ADR-0040) events by event (proposed/reverted/\ + rail_refused) and, for a refusal, reason. Propose-only — no \ + applied event exists.", ) .build(), } @@ -258,9 +270,10 @@ impl EngineMetrics { .add(1, &[opentelemetry::KeyValue::new("state", state)]); } - /// Record one `ContainNode` actuation event (ADR-0040): `event` is one of - /// `proposed`/`applied`/`reverted`/`rail_refused`; `reason` is `Some` only for - /// `rail_refused` — the refusal-reason label + /// Record one `ContainNode` proposal-lifecycle event (ADR-0040 §5): `event` is one of + /// `proposed`/`reverted`/`rail_refused` — there is deliberately no `applied` event, a + /// node cut is propose-first by construction and never auto-applies; `reason` is + /// `Some` only for `rail_refused` — the refusal-reason label /// (`respond::actuator::node_containment::RailRefusal::metric_reason`) alert rules key /// on. Fires unconditionally — this counter carries no arming/mode gate of its own, so a /// `rail_refused` event is exactly as countable in shadow as it would be once a `node` @@ -332,9 +345,9 @@ mod tests { /// ticket adds (with and without a reason) must not panic. #[test] fn record_contain_node_fires_every_event_with_no_armed_state_required() { + // Propose-only (ADR-0040 §5): no "applied" event exists at all. let metrics = EngineMetrics::new(); metrics.record_contain_node("proposed", None); - metrics.record_contain_node("applied", None); metrics.record_contain_node("reverted", None); for reason in [ "control-plane", @@ -342,6 +355,7 @@ mod tests { "worker-floor", "unlabelled", "not-owned", + "unknown-node", ] { metrics.record_contain_node("rail_refused", Some(reason)); } diff --git a/engine/src/engine/mod.rs b/engine/src/engine/mod.rs index d5ca93f..57e76cd 100644 --- a/engine/src/engine/mod.rs +++ b/engine/src/engine/mod.rs @@ -99,6 +99,7 @@ use observe::health::{Health, PodStatusHealth}; use respond::Mitigation; use respond::MitigationLedger; use respond::ProposedAction; +use respond::actuator::node_containment::{self, ProposalOutcome}; use respond::actuator::{ ActionLog, ActuationScope, Actuator, Decision, EnabledActions, decide, predict_blast_radius, }; @@ -289,19 +290,24 @@ pub struct Engine { /// self-revert loop below can hold either the real cluster actuator or a test double — /// mirroring [`Self::actuator`] above. `None` (the [`Self::new`] default) leaves a /// standing `ContainNode` mitigation un-reverted rather than driving it through the - /// wrong-shaped generic `actuator` (see [`Self::revert_contain_node`]); wiring a real - /// one in is a follow-up once a node-observation adapter exists - /// ([`respond::actuator::node_containment`]'s own doc). + /// wrong-shaped generic `actuator` (see [`Self::revert_contain_node`]). Wired to the + /// real cluster actuator in `run_loop.rs` now that the node observation adapter below + /// exists. **There is still NO apply-side actuator field** — ADR-0040 §5 is explicit + /// that a node cut is propose-first BY CONSTRUCTION (alive collateral always present), + /// so arming `ContainNode` (the `node` rung) only makes it ELIGIBLE to be proposed; it + /// is never auto-applied, so there is nothing here for an apply call to reach through. node_containment_actuator: Option>, /// Observed [`respond::actuator::node_containment::NodeFact`]s for the `ContainNode` - /// revert ownership self-gate (ADR-0040 §5), keyed by node name. Empty (the - /// [`Self::new`] default) until seeded via [`Self::with_node_fact`] — a real - /// node-observation adapter refreshing this fleet every pass is a follow-up - /// (`respond::actuator::node_containment`'s own doc). A `ContainNode` mitigation whose - /// host has no entry here is left un-reverted rather than fabricating "no data ⇒ pass" - /// for the ownership check — the same discipline that module already applies to the - /// cordon rails. + /// revert ownership self-gate (ADR-0040 §5) AND the proposal-side rails + /// (`respond::actuator::node_containment::evaluate_proposal`), keyed by node name. + /// Refreshed every pass in [`Self::process`] from the node observation adapter + /// (`observe::adapter::node_fact::observe_node_facts`) over `Snapshot::nodes` — the + /// `nodes` `get/list/watch` RBAC this class's chart wiring always grants. Also + /// seedable directly via [`Self::with_node_fact`] for a test that doesn't drive a real + /// `Snapshot::nodes` watch. A `ContainNode` mitigation whose host has no entry here is + /// left un-reverted / never proposed rather than fabricating "no data ⇒ pass" — the + /// same discipline this module already applies to the cordon rails. node_facts: std::collections::BTreeMap, } @@ -680,18 +686,29 @@ impl Engine { .map(|m| m.cut_signature()) .collect(); // ADR-0040 actuation metrics: a newly-proposed `ContainNode` mitigation is real, - // genuine data today (the `boundary_break` trigger + menu resolver already run - // unconditionally, ADR-0040 §1-3) — unlike the deterministic rails - // (`respond::actuator::node_containment::cordon_decision`/`revert_decision`), which - // need an observed `NodeFact` fleet the engine does not watch yet (that module's own - // doc), so evaluating them here would mean gating against fabricated "no data" and - // silently reading as always-pass. `applied`/`reverted`/`rail_refused` wire in once - // that observation lands. + // genuine data (the `boundary_break` trigger + menu resolver already run + // unconditionally, ADR-0040 §1-3). for mitigation in &ledger_delta.proposed { if mitigation.action == ProposedAction::ContainNode { self.metrics.record_contain_node("proposed", None); } } + // Refresh the observed NodeFact fleet (ADR-0040 §3/§6) from this pass's snapshot — + // the SAME map `revert_contain_node`'s ownership self-gate reads (`node_facts`), so + // the proposal-side rails below and the revert path can never disagree about the + // fleet. Only overwritten when the watch actually observed something THIS pass: + // `Snapshot::nodes` is legitimately empty before the reflector's first sync, and + // replacing wholesale on an empty read would silently erase either that transient + // gap or a test's directly-seeded `with_node_fact` fixture. A later pass with a + // real, non-empty fleet becomes authoritative (a decommissioned node's stale fact + // is dropped once the watch reports its removal). + if !snapshot.nodes.is_empty() { + self.node_facts = observe::adapter::node_fact::observe_node_facts(&snapshot.nodes) + .into_iter() + .map(|fact| (fact.name.clone(), fact)) + .collect(); + } + let node_fleet: Vec<_> = self.node_facts.values().cloned().collect(); // The break-glass kill switch (ADR-0021's enforcement gate, fast path): checked fresh // every pass, narrowing `self.active` down for THIS pass alone when engaged. See @@ -752,6 +769,45 @@ impl Engine { } } } + // ADR-0040 §5/§6: `ContainNode`'s OWN PROPOSAL-side gate — never an apply + // gate. `decide()` above always Forbids it (`is_additive_live() == false`, + // the generic additive-object auto-apply path never applies here) and that + // is correct and permanent: ADR-0040 §5 is explicit that a node cut is + // propose-first BY CONSTRUCTION ("a real node always has alive collateral, + // so the existing blast/alive-collateral gate routes every node cut to human + // approval even at the armed rung") — the deterministic rails below are a + // BOUND layered on top of that human gate, never a replacement for it, so + // there is no cluster write anywhere on this path, armed or not. What arming + // the `node` rung does is make a rails-clean cut ELIGIBLE to be surfaced as + // an actionable proposal instead of merely a structural "not auto-enabled" + // line. `None` (not yet armed/in-scope/corroborated) records nothing new; + // the `Forbidden` arm above already explains that case. Fail closed: a + // target absent from the fleet refuses rather than fabricating a passing + // rail (`node_containment::evaluate_proposal`'s doc). + let armed = effective_active.is_enabled(ProposedAction::ContainNode); + // `ActuationScope::in_scope` resolves a namespace per endpoint via + // `workload_namespace`, which returns `None` for a `host/` key — + // ContainNode's own cut is exactly that, a host self-reference, so checking + // it directly is vacuously in-scope regardless of `enforceScope` (ADR-0021's + // no-wildcard invariant would otherwise not bind this class at all). + // `contain_node_in_scope` confines it through the co-resident LABELLED + // workload set instead, which structurally includes whichever workload's + // `boundary_break` triggered the escalation. + let in_scope = if mitigation.action == ProposedAction::ContainNode { + node_containment::contain_node_in_scope(&graph, &mitigation.cut.from, &self.scope) + } else { + self.scope.in_scope(mitigation) + }; + match node_containment::evaluate_proposal(mitigation, armed, in_scope, &node_fleet) { + None => {} + Some(ProposalOutcome::Proposed) => { + self.metrics.record_contain_node("proposed", None); + } + Some(ProposalOutcome::Refuse(refusal)) => { + self.metrics + .record_contain_node("rail_refused", Some(refusal.metric_reason())); + } + } blasts.push(blast); } self.scope_preview diff --git a/engine/src/engine/node_containment_revert.rs b/engine/src/engine/node_containment_revert.rs index 496e1f2..04f8b7c 100644 --- a/engine/src/engine/node_containment_revert.rs +++ b/engine/src/engine/node_containment_revert.rs @@ -4,9 +4,10 @@ //! [`Engine::revert_contain_node`] — the call `Engine::process`'s self-revert loop makes for //! a standing `ContainNode` reversion instead of the generic network `actuator`. Extracted //! from the orchestrator purely to keep every file under the 1,000-line cap (repo -//! CLAUDE.md); this is a behavior-neutral code move, not a design change — see -//! `respond::actuator::node_containment`'s own doc for why the revert side is wired now -//! while the apply side (the `node` arming rung, node observation, RBAC) is a follow-up. +//! CLAUDE.md). [`Engine::process`] refreshes `node_facts` every pass from the node +//! observation adapter (`observe::adapter::node_fact::observe_node_facts`) now that it +//! exists; [`Self::with_node_fact`] remains for a test that seeds the map directly instead +//! of driving a real `Snapshot::nodes` watch. use super::{Engine, Mitigation, graph, respond}; @@ -28,12 +29,12 @@ impl Engine { self } - /// Seed one observed [`respond::actuator::node_containment::NodeFact`] for the - /// `ContainNode` revert ownership self-gate (ADR-0040 §5), keyed by its own - /// `name`. Builder-style, chainable per node. A real node-observation adapter - /// refreshing this fleet every pass is a follow-up - /// (`respond::actuator::node_containment`'s own doc); until then the map is exactly - /// what a caller (today, only tests) seeded it with. + /// Seed one observed [`respond::actuator::node_containment::NodeFact`] directly, keyed + /// by its own `name` — for a test that exercises the revert/proposal rails without + /// driving a real `Snapshot::nodes` watch. Builder-style, chainable per node. + /// [`Engine::process`] refreshes the map wholesale from the real observation adapter + /// once `Snapshot::nodes` is non-empty (see its own doc), so a seeded fact here only + /// persists across passes that observe no real Node fleet at all. pub fn with_node_fact(mut self, fact: respond::actuator::node_containment::NodeFact) -> Self { self.node_facts.insert(fact.name.clone(), fact); self diff --git a/engine/src/engine/observe/adapter/mod.rs b/engine/src/engine/observe/adapter/mod.rs index a9ef5df..60e024e 100644 --- a/engine/src/engine/observe/adapter/mod.rs +++ b/engine/src/engine/observe/adapter/mod.rs @@ -22,6 +22,10 @@ //! - [`PlacementAdapter`] — `scheduled-on` edges from every scheduled Workload to //! its `Host` (ADR-0040), the pod→node placement fact node containment resolves //! co-residency from. +//! - [`node_fact::observe_node_facts`] — the node-containment rails' `NodeFact` fleet +//! (ADR-0040 §3/§6), mapped from the watched `Node` fleet. NOT an [`Adapter`] (its +//! output bypasses the graph by design, per its own module doc), so it is a free +//! function rather than a struct in this list. use std::collections::{BTreeMap, HashSet}; use std::time::SystemTime; @@ -49,6 +53,7 @@ mod findings; mod ingress_exposure; mod linkerd; mod network; +pub mod node_fact; mod placement; mod rbac; mod secret_mount; diff --git a/engine/src/engine/observe/adapter/node_fact.rs b/engine/src/engine/observe/adapter/node_fact.rs new file mode 100644 index 0000000..4573ee1 --- /dev/null +++ b/engine/src/engine/observe/adapter/node_fact.rs @@ -0,0 +1,319 @@ +//! The node-containment rails' fleet observation (ADR-0040 §3/§6): a metadata-only +//! mapping from the watched `Node` fleet to the +//! [`node_containment::NodeFact`](crate::engine::respond::actuator::node_containment::NodeFact) +//! shape [`cordon_decision`](crate::engine::respond::actuator::node_containment::cordon_decision)/ +//! [`evaluate_proposal`](crate::engine::respond::actuator::node_containment::evaluate_proposal) +//! consult. +//! +//! Mirrors [`super::placement::PlacementAdapter`] in spirit (a small, single-purpose +//! observation of one Kubernetes kind) but NOT in shape: [`observe_node_facts`] does not +//! implement the graph-contributing [`super::Adapter`] trait, because +//! [`NodeFact`](crate::engine::respond::actuator::node_containment::NodeFact) is +//! deliberately NOT the graph's [`crate::engine::graph::Node::Host`] (that module's own +//! doc) — it stays a plain, out-of-graph data type so the rail predicates remain pure and +//! testable over hand-built fixtures without a full [`crate::engine::graph::SecurityGraph`]. +//! This function is [`crate::engine::Engine::process`]'s direct source for the fleet it +//! passes to the rails, built fresh each pass from [`crate::engine::observe::Snapshot::nodes`]. +//! +//! **Reads exactly five fields per `Node` and nothing else** — the metadata-only +//! discipline the module doc promises: the name; the canonical +//! `node-role.kubernetes.io/control-plane` label, the legacy +//! `node-role.kubernetes.io/master` label, AND `spec.taints` (any value, including +//! absent-but-keyed, counts for the labels — kubeadm sets them to the empty string; see +//! [`is_control_plane`] for why all three are checked); `spec.unschedulable`; and whether +//! [`CORDON_OWNER_ANNOTATION`] is set to exactly [`CORDON_OWNER_VALUE`]. In particular it +//! never reads `status` (conditions, capacity, allocatable, images, kubelet version) — +//! there is no sensitive `.data` on a `Node` the way there is on a `Secret`, but the rails +//! have no use for that operational detail either, so it is left unread. + +use k8s_openapi::api::core::v1::Node; + +use crate::engine::respond::actuator::node_containment::{ + CORDON_OWNER_ANNOTATION, CORDON_OWNER_VALUE, NodeFact, +}; + +/// The label kubeadm (and every major managed-Kubernetes control plane) sets on a +/// control-plane node — the primary source of [`NodeFact::control_plane`] (VISION: +/// protector cannot touch the control plane, ADR-0040 §5). Also the taint KEY kubeadm +/// applies alongside the label (`NoSchedule`) — [`is_control_plane`] checks it as its own, +/// independent signal, since an operator can taint without labeling (or vice versa). +const CONTROL_PLANE_LABEL: &str = "node-role.kubernetes.io/control-plane"; + +/// The legacy pre-1.20 spelling of [`CONTROL_PLANE_LABEL`]/its taint — some distros and +/// long-lived clusters still carry `master` rather than `control-plane`. Checked as an +/// equally authoritative signal, never treated as inferior to the canonical spelling. +const LEGACY_CONTROL_PLANE_LABEL: &str = "node-role.kubernetes.io/master"; + +/// Whether `node` carries ANY recognized control-plane signal — the canonical label, the +/// legacy label, or a taint keyed on either (regardless of effect: a control-plane-shaped +/// taint key is itself the signal, not specifically its `NoSchedule` effect). FAIL CLOSED +/// by union, not intersection: a node matching just ONE of these is control-plane, full +/// stop — a heterogeneous or partially-migrated cluster (legacy label only, or a taint +/// applied without the modern label) must never be misread as a plain worker. A node +/// carrying NONE of these signals is treated as an ordinary, cordon-eligible worker — the +/// sound default for vanilla/kubeadm clusters, where every worker legitimately carries no +/// `node-role.kubernetes.io/*` marker at all (only control-plane nodes are labeled); the +/// deterministic rails' own worker-floor/one-node-cap remain the backstop for any +/// residual misclassification this narrower, three-signal check still misses. +fn is_control_plane(node: &Node) -> bool { + let labelled = node.metadata.labels.as_ref().is_some_and(|labels| { + labels.contains_key(CONTROL_PLANE_LABEL) || labels.contains_key(LEGACY_CONTROL_PLANE_LABEL) + }); + let tainted = node + .spec + .as_ref() + .and_then(|spec| spec.taints.as_ref()) + .is_some_and(|taints| { + taints.iter().any(|taint| { + taint.key == CONTROL_PLANE_LABEL || taint.key == LEGACY_CONTROL_PLANE_LABEL + }) + }); + labelled || tainted +} + +/// Map the observed `Node` fleet to the [`NodeFact`] shape the node-containment rails +/// consult. Pure and total: every `Node` in `nodes` yields exactly one `NodeFact`, in the +/// same order, so an empty/absent watch simply yields an empty fleet (the rails then fail +/// closed on any host they can't find, per `evaluate_proposal`'s doc — never fabricate a +/// passing rail from a missing fact). +pub fn observe_node_facts(nodes: &[Node]) -> Vec { + nodes + .iter() + .filter_map(|node| { + let name = node.metadata.name.clone()?; + let control_plane = is_control_plane(node); + let schedulable = !node + .spec + .as_ref() + .and_then(|spec| spec.unschedulable) + .unwrap_or(false); + let owned_by_protector = node + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(CORDON_OWNER_ANNOTATION)) + .is_some_and(|value| value == CORDON_OWNER_VALUE); + Some(NodeFact { + name, + control_plane, + schedulable, + owned_by_protector, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn node(value: serde_json::Value) -> Node { + serde_json::from_value(value).expect("valid Node fixture") + } + + #[test] + fn a_plain_worker_node_maps_to_a_schedulable_unowned_non_control_plane_fact() { + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "node-1"}, + "spec": {} + })); + let facts = observe_node_facts(&[n]); + assert_eq!( + facts, + vec![NodeFact { + name: "node-1".to_string(), + control_plane: false, + schedulable: true, + owned_by_protector: false, + }] + ); + } + + #[test] + fn a_control_plane_labelled_node_is_marked_control_plane() { + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "cp-1", "labels": {"node-role.kubernetes.io/control-plane": ""}}, + "spec": {} + })); + let facts = observe_node_facts(&[n]); + assert!( + facts[0].control_plane, + "the control-plane label must be recognized" + ); + } + + #[test] + fn the_legacy_master_label_is_also_control_plane() { + // Some distros/long-lived clusters still carry the pre-1.20 `master` spelling + // instead of `control-plane` — it must be an equally authoritative signal, not a + // second-class one, or a legacy-labelled control plane is misclassified as a + // cordon-eligible worker. + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "cp-1", "labels": {"node-role.kubernetes.io/master": ""}}, + "spec": {} + })); + let facts = observe_node_facts(&[n]); + assert!( + facts[0].control_plane, + "the legacy master label must be recognized" + ); + } + + #[test] + fn a_control_plane_taint_with_no_label_at_all_is_still_control_plane() { + // Fail closed: a control-plane-shaped taint is its own signal, independent of + // whether the node also carries the label — an operator who taints without + // labeling (or a label that was stripped) must not slip through as a worker. + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "cp-1"}, + "spec": { + "taints": [ + {"key": "node-role.kubernetes.io/control-plane", "effect": "NoSchedule"} + ] + } + })); + let facts = observe_node_facts(&[n]); + assert!( + facts[0].control_plane, + "a control-plane taint alone must mark the node control-plane" + ); + } + + #[test] + fn a_legacy_master_taint_is_also_recognized() { + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "cp-1"}, + "spec": { + "taints": [{"key": "node-role.kubernetes.io/master", "effect": "NoSchedule"}] + } + })); + let facts = observe_node_facts(&[n]); + assert!(facts[0].control_plane); + } + + #[test] + fn an_unrelated_taint_does_not_mark_a_node_control_plane() { + // Only a control-plane-shaped taint KEY counts — an ordinary workload taint + // (dedicated-node-pool style) must not false-positive a worker into + // control-plane (which would make it un-cordonable but is not the fail-closed + // direction this check protects — it is checked for completeness alongside the + // fail-closed cases above). + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "node-1"}, + "spec": { + "taints": [{"key": "dedicated", "value": "gpu", "effect": "NoSchedule"}] + } + })); + let facts = observe_node_facts(&[n]); + assert!(!facts[0].control_plane); + } + + #[test] + fn a_node_with_no_role_signal_at_all_is_the_ordinary_cordon_eligible_worker_default() { + // Vanilla/kubeadm workers carry NO node-role label or taint at all — that is the + // expected, sound "worker" case, not a gap: only a POSITIVE control-plane signal + // (label or taint, canonical or legacy) ever excludes a node here. + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "node-1"}, + "spec": {} + })); + let facts = observe_node_facts(&[n]); + assert!(!facts[0].control_plane); + } + + #[test] + fn spec_unschedulable_maps_to_not_schedulable() { + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "node-1"}, + "spec": {"unschedulable": true} + })); + let facts = observe_node_facts(&[n]); + assert!(!facts[0].schedulable); + } + + #[test] + fn the_cordon_ownership_annotation_maps_to_owned_by_protector() { + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": { + "name": "node-1", + "annotations": {"protector.jeffl.es/cordoned-by": "protector"} + }, + "spec": {"unschedulable": true} + })); + let facts = observe_node_facts(&[n]); + assert!(facts[0].owned_by_protector); + } + + #[test] + fn a_foreign_cordon_annotation_value_does_not_count_as_protector_owned() { + // A human/autoscaler cordon carries no protector annotation at all in practice, + // but even a look-alike key with a DIFFERENT value must not be read as ours — + // exact-value match only (mirrors `cordon_decision`'s ownership discipline). + let n = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": { + "name": "node-1", + "annotations": {"protector.jeffl.es/cordoned-by": "someone-else"} + }, + "spec": {"unschedulable": true} + })); + let facts = observe_node_facts(&[n]); + assert!(!facts[0].owned_by_protector); + } + + #[test] + fn a_node_with_no_name_is_skipped() { + // Every real `Node` from the apiserver carries a name; this only guards the + // theoretical malformed case rather than panicking. + let n = node(json!({"apiVersion": "v1", "kind": "Node", "metadata": {}, "spec": {}})); + assert!(observe_node_facts(&[n]).is_empty()); + } + + #[test] + fn maps_the_whole_fleet_in_order() { + let a = node(json!({ + "apiVersion": "v1", "kind": "Node", "metadata": {"name": "a"}, "spec": {} + })); + let b = node(json!({ + "apiVersion": "v1", "kind": "Node", "metadata": {"name": "b"}, "spec": {} + })); + let facts = observe_node_facts(&[a, b]); + let names: Vec<_> = facts.iter().map(|f| f.name.as_str()).collect(); + assert_eq!(names, vec!["a", "b"]); + } + + #[test] + fn status_fields_are_never_read_regardless_of_content() { + // A rich `status` block (images, conditions, capacity/allocatable) must not + // change the mapped fact at all — the metadata-only discipline this module's + // doc promises. Compare against the same node with no status. + let bare = node(json!({ + "apiVersion": "v1", "kind": "Node", "metadata": {"name": "node-1"}, "spec": {} + })); + let with_status = node(json!({ + "apiVersion": "v1", "kind": "Node", + "metadata": {"name": "node-1"}, + "spec": {}, + "status": { + "conditions": [{"type": "Ready", "status": "True"}], + "capacity": {"cpu": "8", "memory": "32Gi"}, + "allocatable": {"cpu": "7500m", "memory": "30Gi"}, + "images": [{"names": ["example.com/some-image:1"], "sizeBytes": 12345}] + } + })); + assert_eq!( + observe_node_facts(&[bare]), + observe_node_facts(&[with_status]) + ); + } +} diff --git a/engine/src/engine/observe/mod.rs b/engine/src/engine/observe/mod.rs index 2442f75..cba9c55 100644 --- a/engine/src/engine/observe/mod.rs +++ b/engine/src/engine/observe/mod.rs @@ -31,7 +31,7 @@ pub mod trivy_config; pub mod trivy_rbac; pub mod trivy_secret; -use k8s_openapi::api::core::v1::{Pod, Secret, Service}; +use k8s_openapi::api::core::v1::{Node, Pod, Secret, Service}; use k8s_openapi::api::networking::v1::{Ingress, IngressClass, NetworkPolicy}; use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, Role, RoleBinding}; use kube::Api; @@ -273,6 +273,13 @@ pub struct Snapshot { pub linkerd_servers: Vec, pub linkerd_authz_policies: Vec, pub linkerd_mtls_auths: Vec, + /// The cluster's `Node` fleet (ADR-0040 §3/§6) — the raw material + /// [`adapter::node_fact::observe_node_facts`] maps into the node-containment rails' + /// [`crate::engine::respond::actuator::node_containment::NodeFact`] shape. Read-only, + /// always populated (the `nodes` RBAC read is always-on regardless of posture — see + /// the chart's ClusterRole); empty when the watch hasn't synced yet, which the rails + /// then treat as "no fact for this host" and fail closed on. + pub nodes: Vec, } impl Snapshot { @@ -299,6 +306,7 @@ impl Snapshot { image_vulns, trivy_findings, linkerd, + nodes, ) = tokio::try_join!( async { anyhow::Ok(Api::::all(client.clone()).list(&lp).await?.items) }, async { @@ -375,6 +383,10 @@ impl Snapshot { // report — empty when their CRDs are absent, so they never fail the join. async { anyhow::Ok(list_trivy_findings(&client).await) }, async { anyhow::Ok(list_linkerd_authz(&client).await) }, + // The Node fleet (ADR-0040 §3/§6) — the node-containment rails' raw material. + // Always-granted (like every other list here except Ingress), so a failure + // fails the whole join rather than degrading. + async { anyhow::Ok(Api::::all(client.clone()).list(&lp).await?.items) }, )?; let (image_secrets, config_audits, rbac_assessments) = trivy_findings; let (linkerd_servers, linkerd_authz_policies, linkerd_mtls_auths) = linkerd; @@ -413,6 +425,7 @@ impl Snapshot { linkerd_servers, linkerd_authz_policies, linkerd_mtls_auths, + nodes, }) } } diff --git a/engine/src/engine/respond/actuator/arming_ladder.rs b/engine/src/engine/respond/actuator/arming_ladder.rs index 255ec8e..cb5e3d9 100644 --- a/engine/src/engine/respond/actuator/arming_ladder.rs +++ b/engine/src/engine/respond/actuator/arming_ladder.rs @@ -14,9 +14,10 @@ use super::EnabledActions; use crate::engine::respond::ProposedAction; -/// How far up the network-cut severity ladder `mode: enforce` is armed. Ordered: -/// each rung implies its narrower predecessor(s) — [`Quarantine`](Self::Quarantine) -/// still arms the edge-cut, it never replaces it. +/// How far up the cut-severity ladder `mode: enforce` is armed. Ordered: each rung +/// implies its narrower predecessor(s) — [`Quarantine`](Self::Quarantine) still arms +/// the edge-cut, it never replaces it, and [`Node`](Self::Node) still arms both network +/// rungs beneath it. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ArmingRung { /// Rung 1 — the narrowest, most-reversible cut, and the `enforce` default: only the @@ -29,6 +30,22 @@ pub enum ArmingRung { /// ADR-0010) and the compromised-workload quarantine /// ([`QuarantineWorkload`](ProposedAction::QuarantineWorkload)). Quarantine, + /// Rung 3 — the top of the ladder, strictly above [`Quarantine`](Self::Quarantine) + /// (`edge-cut < quarantine < node`, ADR-0040 §6): also arms + /// [`ContainNode`](ProposedAction::ContainNode), the node-scoped escalation of a + /// model-named workload whose own evidence proves its pod boundary broken. + /// Protector's first NODE-write RBAC class — a cordon plus a co-resident + /// default-deny sweep — so it is a deliberate, explicit third opt-in, never implied + /// by `quarantine`. **Arming this rung never makes a cordon happen.** ADR-0040 §5 + /// makes a node cut propose-first BY CONSTRUCTION (a real node always has alive + /// collateral) — `ContainNode` has no auto-apply path at any rung, armed or not. + /// What this rung does is make a cut whose deterministic rails (control-plane + /// exclusion, one-node cap, the two-worker floor, ownership-gated revert — + /// `respond::actuator::node_containment::cordon_decision`) all pass ELIGIBLE to + /// surface as an actionable proposal (`respond::actuator::node_containment::evaluate_proposal`) + /// instead of a bare "not auto-enabled" line; a human out-of-band action is the only + /// route to an actual cordon. + Node, } impl ArmingRung { @@ -38,6 +55,7 @@ impl ArmingRung { pub fn from_name(name: &str) -> Self { match name.trim() { "quarantine" => Self::Quarantine, + "node" => Self::Node, _ => Self::EdgeCut, } } @@ -46,11 +64,15 @@ impl ArmingRung { /// always include every action their narrower predecessors arm. pub fn enabled_actions(self) -> EnabledActions { let armed = EnabledActions::none().enable(ProposedAction::DenyNetworkPath); + let quarantines = |armed: EnabledActions| { + armed + .enable(ProposedAction::QuarantineEntry) + .enable(ProposedAction::QuarantineWorkload) + }; match self { Self::EdgeCut => armed, - Self::Quarantine => armed - .enable(ProposedAction::QuarantineEntry) - .enable(ProposedAction::QuarantineWorkload), + Self::Quarantine => quarantines(armed), + Self::Node => quarantines(armed).enable(ProposedAction::ContainNode), } } } @@ -84,13 +106,51 @@ mod tests { ArmingRung::from_name(" quarantine "), ArmingRung::Quarantine ); + assert_eq!(ArmingRung::from_name(" node "), ArmingRung::Node); + } + + #[test] + fn node_rung_implies_the_quarantine_and_edge_cut_rungs_and_adds_contain_node() { + // ADR-0040 §6: `node` is strictly above `quarantine` — arming it still implies + // every narrower rung's actions, plus the new node-write class. + let armed = ArmingRung::Node.enabled_actions(); + assert!(armed.is_enabled(ProposedAction::DenyNetworkPath)); + assert!(armed.is_enabled(ProposedAction::QuarantineEntry)); + assert!(armed.is_enabled(ProposedAction::QuarantineWorkload)); + assert!(armed.is_enabled(ProposedAction::ContainNode)); + } + + #[test] + fn only_the_node_rung_arms_contain_node() { + // Neither lower rung may arm the node-write class — it is a deliberate, explicit + // third opt-in, never implied by `edge-cut`/`quarantine` (ADR-0040 §6). + assert!( + !ArmingRung::EdgeCut + .enabled_actions() + .is_enabled(ProposedAction::ContainNode) + ); + assert!( + !ArmingRung::Quarantine + .enabled_actions() + .is_enabled(ProposedAction::ContainNode) + ); + assert!( + ArmingRung::Node + .enabled_actions() + .is_enabled(ProposedAction::ContainNode) + ); } #[test] - fn neither_rung_arms_a_non_network_action_class() { - // The ladder only ever governs the two live-actuatable network classes — no - // rung enables a subtractive/irreversible class regardless of position. - for rung in [ArmingRung::EdgeCut, ArmingRung::Quarantine] { + fn no_rung_arms_a_subtractive_or_irreversible_action_class() { + // The ladder only ever governs the live-actuatable classes (the two network + // denies, and now ContainNode) — no rung, including the top one, enables a + // subtractive/irreversible class. + for rung in [ + ArmingRung::EdgeCut, + ArmingRung::Quarantine, + ArmingRung::Node, + ] { let armed = rung.enabled_actions(); assert!(!armed.is_enabled(ProposedAction::RevokeRbacGrant)); assert!(!armed.is_enabled(ProposedAction::RemoveSecretMount)); diff --git a/engine/src/engine/respond/actuator/mod.rs b/engine/src/engine/respond/actuator/mod.rs index 8f42fc9..f3f072f 100644 --- a/engine/src/engine/respond/actuator/mod.rs +++ b/engine/src/engine/respond/actuator/mod.rs @@ -183,6 +183,15 @@ impl ActuationScope { Self { namespaces, labels } } + /// Whether NO `enforceScope` is configured (both axes empty) — the historical, + /// shadow-default meaning "every namespace eligible", not "matches nothing". Exposed + /// for a caller (`node_containment::contain_node_in_scope`) that can't route a + /// non-workload cut through [`Self::in_scope`]'s own per-mitigation shortcut and needs + /// the same "is a scope even configured" check directly. + pub fn is_unscoped(&self) -> bool { + self.namespaces.is_empty() && self.labels.is_empty() + } + /// Whether one workload endpoint (its namespace + labels) is in scope: its namespace /// is listed, **or** it carries one of the scoped labels. Mirrors `EnforceScope`'s /// namespace-OR-label match so the engine cut and the webhook gates agree. diff --git a/engine/src/engine/respond/actuator/node_containment.rs b/engine/src/engine/respond/actuator/node_containment.rs index b071b65..c52d3ab 100644 --- a/engine/src/engine/respond/actuator/node_containment.rs +++ b/engine/src/engine/respond/actuator/node_containment.rs @@ -1,20 +1,7 @@ -//! The [`ProposedAction::ContainNode`] actuator (ADR-0040 §4/§5): the cordon + co-resident -//! default-deny rendering, and the deterministic rails that gate it. Split out of the -//! actuator module root purely to keep every file under the 1,000-line cap (repo CLAUDE.md). -//! -//! **The apply side is unit-tested, wired nowhere live yet; the revert side IS wired into -//! `Engine::process`'s break-glass/self-revert loop.** `ContainNode` is -//! `is_additive_live() == false` ([`ProposedAction::is_additive_live`]), so -//! [`super::decide`] already routes every `ContainNode` mitigation to -//! [`super::Decision::Forbidden`] regardless of what these rails would say — there is no -//! `node` arming rung to escalate past (ADR-0040 §6, a separate ticket), so nothing here can -//! become live-*applied* through this module alone. But ADR-0040 §5 also requires -//! `ContainNode` to join the armed-set revert trigger (ADR-0036) and the standard ledger -//! self-revert (ADR-0017) — that half does not depend on the apply-side rung existing at -//! all, so it is wired now: `Engine::process`'s self-revert loop routes a standing -//! `ContainNode` reversion through [`NodeContainmentRevert`] rather than the generic network -//! `actuator`, whose `revert()` speaks a different object shape entirely and would silently -//! leave the node cordoned. What IS delivered: +//! The [`ProposedAction::ContainNode`] actuator (ADR-0040 §4/§5/§6): the cordon + +//! co-resident default-deny rendering, and the deterministic rails that gate it. Split out +//! of the actuator module root purely to keep every file under the 1,000-line cap (repo +//! CLAUDE.md). //! //! - [`render_cordon`]/[`render_uncordon`]: the pure `Node.spec.unschedulable` patch, //! carrying [`CORDON_OWNER_ANNOTATION`] so a revert only ever lifts a cordon protector @@ -28,31 +15,62 @@ //! exclusion, one-node cap, the two-worker floor, ownership-gated revert), pure over a //! [`NodeFact`] fleet so they're unit-testable without a live cluster and independent of //! any arming/enabled state — a rail refusal is exactly as meaningful in shadow as it -//! would be armed. +//! would be armed. These rails are a BOUND layered on top of the human-approval gate +//! below, never a substitute for it. +//! - [`contain_node_in_scope`]: the `enforceScope` confinement check +//! [`crate::engine::Engine::process`] uses in place of the generic +//! [`super::ActuationScope::in_scope`] — `ContainNode`'s own cut is a `host/` +//! self-reference, which the generic check can't resolve a namespace for and so treats +//! as vacuously in scope; this instead confines through the co-resident LABELLED +//! workload set (ADR-0021: no enforce-everywhere). +//! - [`evaluate_proposal`]: the PROPOSAL-side gate [`crate::engine::Engine::process`] calls +//! for every active `ContainNode` mitigation each pass — armed/in-scope/ +//! live-corroborated (mirroring [`super::decide`]'s own ordering), THEN the +//! [`cordon_decision`] rail against the pass's observed [`NodeFact`] fleet, fail-closed on +//! a target absent from the fleet ([`RailRefusal::UnknownNode`]) rather than fabricating a +//! passing rail. Pure — the caller maps its result onto the `proposed`/`rail_refused` +//! metric. **There is no `evaluate_apply` and no cluster write here**: ADR-0040 §5 is +//! explicit that a node cut is propose-first BY CONSTRUCTION ("a real node always has +//! alive collateral, so the existing blast/alive-collateral gate routes every node cut to +//! human approval even at the armed rung; propose-first is structural, not a toggle") — +//! these rails are bounds a human reviews the proposal against, never an auto-apply gate. //! - [`live`]'s [`NodeContainmentActuator`]/[`NodeContainmentRevert`]: the cluster-facing -//! apply/revert glue. Thin and untested against a real cluster, like +//! REVERT glue only (lifting an already-standing cut via break-glass/self-revert, +//! `crate::engine::node_containment_revert`) — there is no corresponding live APPLY call +//! site; `NodeContainmentActuator::apply` exists and is unit-tested in isolation for a +//! future human-approval-to-apply flow (out of scope here), but nothing in `Engine` +//! invokes it. Thin and untested against a real cluster, like //! [`super::KubeActuator`]/[`super::IsolationActuator`] — [`render_cordon`]/ -//! [`render_uncordon`] are the unit-tested pure half. +//! [`render_uncordon`]/[`evaluate_proposal`]/[`contain_node_in_scope`] are the unit-tested +//! pure half. +//! +//! `ContainNode` is `is_additive_live() == false` ([`ProposedAction::is_additive_live`]), so +//! [`super::decide`]'s generic AutoApply path always routes it to +//! [`super::Decision::Forbidden`] — a cordon mutates a shared field on a live object rather +//! than adding a new engine-owned one, so it can never ride the generic additive-object +//! auto-apply path, and (unlike every other class) arming the `node` rung +//! ([`super::arming_ladder::ArmingRung::Node`]) does not change that: [`evaluate_proposal`] +//! only ever answers "should this be SURFACED as an actionable proposal", never "should +//! this be applied". //! -//! **Node role/schedulability observation is a follow-up, not this ticket.** [`NodeFact`] -//! is the fleet-state shape the rails need, but nothing in the engine watches Kubernetes -//! `Node` objects today — only `Pod.spec.nodeName`-derived placement (the placement -//! adapter, ADR-0040 §3), which needs no new RBAC. Populating a -//! real `NodeFact` fleet needs a `nodes` `get/list/watch` grant this ticket deliberately -//! does not add (ADR-0040 §7 ships the actuator split from the chart/RBAC change; the -//! ticket that adds this observation is the natural place to also wire the apply side of -//! these rails into `Engine::process`'s per-pass loop, and to keep `Engine`'s attached -//! `NodeFact` fleet fresh every pass). Evaluating a rail against a fabricated "no data" -//! fleet would silently default it to PASS — exactly the "weakening the rail" the ADR's -//! build-settled note warns against — so the engine's self-revert loop skips (rather than -//! fabricates) a revert for any host with no attached [`NodeContainmentRevert`] or no -//! observed [`NodeFact`], the same discipline this doc already applied to the cordon rails. +//! **Node role/schedulability observation** (ADR-0040 §3, `node_fact` fleet): a metadata-only +//! `Node` watch (name; the `node-role.kubernetes.io/control-plane` label, the legacy +//! `node-role.kubernetes.io/master` label, and a control-plane-shaped `NoSchedule` taint; +//! `spec.unschedulable`; [`CORDON_OWNER_ANNOTATION`]) — +//! [`crate::engine::observe::adapter::node_fact::observe_node_facts`] — is the sole source of +//! the [`NodeFact`] fleet [`evaluate_proposal`]/[`cordon_decision`] consult. See that +//! module's doc for why it does not implement the graph-contributing +//! [`crate::engine::observe::adapter::Adapter`] trait `PlacementAdapter` does: [`NodeFact`] +//! deliberately bypasses the [`crate::engine::graph::SecurityGraph`] entirely, to keep the +//! rails pure and testable without one. use crate::engine::graph::{NodeKey, SecurityGraph}; use crate::engine::respond::{ Mitigation, ProposedAction, co_resident_workloads, quarantine_workload_link, }; +use super::ActuationScope; + mod live; pub use live::{NodeContainmentActuator, NodeContainmentRevert}; @@ -132,20 +150,31 @@ pub fn co_resident_denies(graph: &SecurityGraph, host: &NodeKey) -> Vec "worker-floor", Self::Unlabelled => "unlabelled", Self::NotOwned => "not-owned", + Self::UnknownNode => "unknown-node", } } } -/// The minimum number of schedulable, non-control-plane workers a cordon must leave behind -/// (ADR-0040 §5, build-settled 2026-08-02: "a floor that leaves a single worker is an -/// outage, not damage-limitation" — kept at 2 even on a small fleet where this can make -/// `ContainNode` correctly, permanently inert). +/// The minimum number of SCHEDULABLE (not necessarily Ready — [`NodeFact::schedulable`]'s +/// own doc), non-control-plane workers a cordon must leave behind (ADR-0040 §5, +/// build-settled 2026-08-02: "a floor that leaves a single worker is an outage, not +/// damage-limitation" — kept at 2 even on a small fleet where this can make `ContainNode` +/// correctly, permanently inert). const WORKER_FLOOR: usize = 2; /// Whether cordoning `target` is deterministically allowed, over the CURRENT `fleet` @@ -233,5 +269,96 @@ pub fn revert_decision(target: &NodeFact) -> Result<(), RailRefusal> { } } +/// Whether a `ContainNode` mitigation targeting `host` is within `scope` (ADR-0021: no +/// enforce-everywhere). Unlike every other mitigation, `ContainNode`'s own cut is a +/// `host/` self-reference — a non-workload key [`ActuationScope::in_scope`] can't +/// resolve a namespace for (`workload_namespace` returns `None`), so checking the cut +/// directly would be vacuously true regardless of `enforceScope`, a real scope bypass: +/// arming the `node` rung would then confine nothing. This confines it instead through the +/// SAME co-resident LABELLED workload set the default-deny sweep already computes +/// ([`co_resident_denies`]) — which structurally includes whichever workload's +/// `boundary_break` triggered the escalation, since it is scheduled on `host` by +/// construction ([`crate::engine::respond::contain_node_link`]'s own doc). +/// +/// In scope iff `scope` is unscoped (the historical, no-`enforceScope`-configured +/// behavior), OR at least one co-resident deny target is itself in scope. A host with no +/// co-resident LABELLED pod at all (nothing for [`co_resident_denies`] to return) is never +/// presumed in scope once a scope IS configured — the same "decline rather than widen" +/// discipline [`co_resident_denies`] already applies to an unlabelled pod. +pub fn contain_node_in_scope( + graph: &SecurityGraph, + host: &NodeKey, + scope: &ActuationScope, +) -> bool { + if scope.is_unscoped() { + return true; + } + co_resident_denies(graph, host) + .iter() + .any(|deny| scope.in_scope(deny)) +} + +/// What [`evaluate_proposal`] decided for one active `ContainNode` mitigation this pass. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProposalOutcome { + /// Every deterministic rail passes: the cut is ELIGIBLE to be surfaced as an + /// actionable proposal. This NEVER means it was applied — ADR-0040 §5 makes a node + /// cut propose-first BY CONSTRUCTION, so there is no cluster write on this path at + /// any rung; a human out-of-band action is the only route to an actual cordon. + Proposed, + /// A deterministic rail refused — includes [`RailRefusal::UnknownNode`] for a target + /// absent from the observed fleet (fail closed). + Refuse(RailRefusal), +} + +/// The proposal-side decision for one active `ContainNode` mitigation this pass (ADR-0040 +/// §5/§6): whether `mitigation` should be surfaced as a rails-clean, actionable proposal, +/// given this pass's arming/scope/corroboration state and the observed [`NodeFact`] fleet. +/// **Never an apply decision** — see this module's doc for why `ContainNode` has no +/// `evaluate_apply` counterpart: ADR-0040 §5 states plainly that "a real node always has +/// alive collateral, so the existing blast/alive-collateral gate routes every node cut to +/// human approval even at the armed rung; propose-first is structural, not a toggle." The +/// rails here are the SAME bound that gate carries — control-plane/one-node/worker-floor — +/// layered on top of the human-approval requirement, never a way to skip it. +/// +/// `None` when the mitigation is not yet eligible to be evaluated against the rails at +/// all — not armed at the `node` rung, out of `enforceScope` +/// ([`contain_node_in_scope`]), or not live-corroborated (mirrors [`super::decide`]'s own +/// ordering for every other action class: the generic `Decision::Forbidden`/`Propose` path +/// above already explains a `None` case, so this function stays silent rather than +/// emitting a competing "reason"). `Some` once eligible: [`ProposalOutcome::Proposed`] if +/// [`cordon_decision`] passes (or the host is unrecognized, in which case this fails +/// closed to [`ProposalOutcome::Refuse`]`(`[`RailRefusal::UnknownNode`]`)` rather than +/// silently defaulting a missing fact to a passing rail — the exact "weakening the rail" +/// failure mode this ADR's build-settled note warns against), otherwise +/// [`ProposalOutcome::Refuse`] with the rail that blocked it. +/// +/// Pure: takes the mitigation, the three upstream gates as bools, and the fleet — no +/// [`SecurityGraph`], no cluster client, no metrics/OTLP — so it is fully unit-testable and +/// the only thing [`Engine::process`](crate::engine::Engine::process) has to do with the +/// result is record the matching metric. There is no further action to take on +/// `Proposed`: it is surfaced, never applied. +pub fn evaluate_proposal( + mitigation: &Mitigation, + armed: bool, + in_scope: bool, + fleet: &[NodeFact], +) -> Option { + if mitigation.action != ProposedAction::ContainNode { + return None; + } + if !armed || !in_scope || !mitigation.is_live_corroborated() { + return None; + } + let host_name = mitigation.cut.from.short(); + let Some(target) = fleet.iter().find(|n| n.name == host_name) else { + return Some(ProposalOutcome::Refuse(RailRefusal::UnknownNode)); + }; + Some(match cordon_decision(target, fleet) { + Ok(()) => ProposalOutcome::Proposed, + Err(refusal) => ProposalOutcome::Refuse(refusal), + }) +} + #[cfg(test)] mod tests; diff --git a/engine/src/engine/respond/actuator/node_containment/live.rs b/engine/src/engine/respond/actuator/node_containment/live.rs index 883f92b..90ec30d 100644 --- a/engine/src/engine/respond/actuator/node_containment/live.rs +++ b/engine/src/engine/respond/actuator/node_containment/live.rs @@ -4,11 +4,12 @@ //! exercised only against a real cluster — like the `kube` module's live actuators — with //! [`super::render_cordon`]/[`super::render_uncordon`] as the unit-tested pure half. //! -//! **Not wired into anything live in this ticket.** No `node` arming rung exists yet -//! (ADR-0040 §6, a separate ticket) and `ContainNode`'s `is_additive_live() == false` means -//! [`super::super::decide`] never routes here through the generic auto-apply path either — -//! this exists so a future human-approval/break-glass-revert path has a real apply/revert to -//! call, callable and testable in isolation today. +//! [`Self::revert`] is the break-glass/self-revert call site's real revert to call +//! ([`crate::engine::node_containment_revert`]), self-gated on [`super::revert_decision`] +//! regardless of caller discipline. **[`Self::apply`] has NO call site in `Engine`** — +//! ADR-0040 §5 makes a node cut propose-first by construction (never auto-applied, at any +//! arming rung), so nothing in the engine's per-pass loop ever reaches it; it exists, +//! tested in isolation, for a future human-approval-to-apply flow. use crate::engine::respond::Mitigation; use crate::engine::respond::actuator::{Actuation, Actuator, IsolationActuator, cut_label}; diff --git a/engine/src/engine/respond/actuator/node_containment/tests.rs b/engine/src/engine/respond/actuator/node_containment/tests.rs index a856cf8..49b4b30 100644 --- a/engine/src/engine/respond/actuator/node_containment/tests.rs +++ b/engine/src/engine/respond/actuator/node_containment/tests.rs @@ -1,11 +1,18 @@ //! Unit tests for the ADR-0040 node-containment actuator: the cordon/uncordon renderers, -//! the co-resident default-deny sweep, and each deterministic rail — plus the standing -//! invariant that `ContainNode` still never auto-applies (no `node` rung wired). +//! the co-resident default-deny sweep, each deterministic rail, the `enforceScope` +//! confinement check (`contain_node_in_scope`), and the PROPOSAL-side gate +//! (`evaluate_proposal`) — including that it stays `None` (never armed) unless the +//! mitigation is armed, in scope, AND live-corroborated; fails closed on a host absent +//! from the observed fleet; and NEVER returns anything an apply call could reach (there is +//! no `ApplyOutcome`/`evaluate_apply` — `ProposalOutcome::Proposed` is the only "eligible" +//! outcome, and it is surfaced, never applied, per ADR-0040 §5). use super::*; +use crate::engine::graph::attack::CREDENTIAL_ACCESS; use crate::engine::observe::Snapshot; use crate::engine::observe::adapter::{build_graph, default_adapters}; use crate::engine::reason::proof::Link; +use crate::engine::respond::Justification; use serde_json::json; /// A `ContainNode` mitigation self-referencing `host/` — the exact shape @@ -26,6 +33,24 @@ fn contain_node_mitigation(host_name: &str) -> Mitigation { } } +/// A `ContainNode` mitigation that ALSO clears the live-corroboration bar +/// (`Mitigation::is_live_corroborated`) — a corroborated, adjudicated, +/// breach-relevant justification, mirroring `respond::tests`' own fixture shape. +fn corroborated_contain_node_mitigation(host_name: &str) -> Mitigation { + let mut m = contain_node_mitigation(host_name); + m.justifications.push(Justification { + entry: "workload/app/Pod/entry".into(), + objective: format!("host/{host_name}"), + attack: CREDENTIAL_ACCESS, + foothold: false, + corroborated: true, + adjudicated: true, + promoted: false, + breach_relevant: true, + }); + m +} + fn scheduled_pod( name: &str, node_name: &str, @@ -151,6 +176,7 @@ fn every_rail_refusal_has_its_specified_metric_reason() { assert_eq!(RailRefusal::WorkerFloor.metric_reason(), "worker-floor"); assert_eq!(RailRefusal::Unlabelled.metric_reason(), "unlabelled"); assert_eq!(RailRefusal::NotOwned.metric_reason(), "not-owned"); + assert_eq!(RailRefusal::UnknownNode.metric_reason(), "unknown-node"); } // --- cordon_decision rails --- @@ -290,3 +316,184 @@ fn rail_decisions_take_no_arming_state_and_so_are_exactly_as_meaningful_in_shado let unowned = fact("node-1", false, false, false); assert_eq!(revert_decision(&unowned), Err(RailRefusal::NotOwned)); } + +// --- contain_node_in_scope: enforceScope confinement (ADR-0021) --- + +#[test] +fn contain_node_in_scope_is_true_when_unscoped() { + // No enforceScope configured at all (both axes empty) is the historical + // "every namespace eligible" meaning, not "matches nothing" — even a host with NO + // co-resident labelled pod at all must read as in scope here. + let graph = crate::engine::graph::SecurityGraph::new(); + let host = NodeKey("host/node-1".into()); + assert!(contain_node_in_scope( + &graph, + &host, + &crate::engine::respond::actuator::ActuationScope::unscoped() + )); +} + +#[test] +fn contain_node_in_scope_matches_a_co_resident_labelled_pods_namespace() { + let snap = Snapshot { + pods: vec![scheduled_pod("victim", "node-1", json!({"app": "victim"}))], + ..Default::default() + }; + let graph = build_graph(&snap, &default_adapters()); + let host = NodeKey("host/node-1".into()); + let scope = + crate::engine::respond::actuator::ActuationScope::enforce_namespaces(["app".to_string()]); + assert!(contain_node_in_scope(&graph, &host, &scope)); +} + +#[test] +fn contain_node_in_scope_is_false_when_no_co_resident_pod_is_in_a_configured_scope() { + // A real, non-empty enforceScope is configured, but the host's only co-resident + // labelled pod lives in a DIFFERENT namespace — checking the host's OWN cut + // directly (the bug this function fixes) would read as vacuously in scope; this + // must not. + let snap = Snapshot { + pods: vec![scheduled_pod("victim", "node-1", json!({"app": "victim"}))], + ..Default::default() + }; + let graph = build_graph(&snap, &default_adapters()); + let host = NodeKey("host/node-1".into()); + let scope = crate::engine::respond::actuator::ActuationScope::enforce_namespaces([ + "payments".to_string() + ]); + assert!(!contain_node_in_scope(&graph, &host, &scope)); +} + +#[test] +fn contain_node_in_scope_is_false_for_a_scoped_host_with_no_labelled_co_resident_pod() { + // A scope IS configured, but the host's only co-resident pod carries no labels at + // all (so `co_resident_denies` declines it, same as everywhere else) — never + // presumed in scope just because there was nothing to check. + let snap = Snapshot { + pods: vec![scheduled_pod("bare", "node-1", json!({}))], + ..Default::default() + }; + let graph = build_graph(&snap, &default_adapters()); + let host = NodeKey("host/node-1".into()); + let scope = + crate::engine::respond::actuator::ActuationScope::enforce_namespaces(["app".to_string()]); + assert!(!contain_node_in_scope(&graph, &host, &scope)); +} + +// --- evaluate_proposal: the PROPOSAL-side gate Engine::process calls (ADR-0040 §5/§6) --- +// NEVER an apply gate — there is no `ApplyOutcome`/apply call path for `ContainNode` at +// all; `ProposalOutcome::Proposed` is surfaced, never acted on. + +#[test] +fn evaluate_proposal_is_none_for_a_non_contain_node_action() { + let other = Mitigation { + cut: Link { + from: NodeKey("workload/app/Pod/web".into()), + to: NodeKey("workload/app/Pod/web".into()), + relation: "quarantine-workload".to_string(), + technique: None, + from_labels: [("app".to_string(), "web".to_string())].into(), + to_labels: [("app".to_string(), "web".to_string())].into(), + }, + action: ProposedAction::QuarantineWorkload, + justifications: vec![], + }; + assert_eq!(evaluate_proposal(&other, true, true, &[]), None); +} + +#[test] +fn evaluate_proposal_is_none_when_not_armed() { + let m = corroborated_contain_node_mitigation("node-1"); + let fleet = vec![ + fact("node-1", false, true, false), + fact("node-2", false, true, false), + ]; + assert_eq!(evaluate_proposal(&m, false, true, &fleet), None); +} + +#[test] +fn evaluate_proposal_is_none_when_out_of_scope() { + let m = corroborated_contain_node_mitigation("node-1"); + let fleet = vec![ + fact("node-1", false, true, false), + fact("node-2", false, true, false), + ]; + assert_eq!(evaluate_proposal(&m, true, false, &fleet), None); +} + +#[test] +fn evaluate_proposal_is_none_when_not_live_corroborated() { + // `contain_node_mitigation` carries no justifications at all — never corroborated. + let m = contain_node_mitigation("node-1"); + let fleet = vec![ + fact("node-1", false, true, false), + fact("node-2", false, true, false), + ]; + assert_eq!(evaluate_proposal(&m, true, true, &fleet), None); +} + +#[test] +fn evaluate_proposal_fails_closed_on_a_host_absent_from_the_fleet() { + // Armed + in-scope + corroborated, but the fleet has no entry for the target host at + // all (the watch hasn't synced, or the host no longer exists) — must refuse, never + // fabricate a passing rail. + let m = corroborated_contain_node_mitigation("node-1"); + let fleet = vec![ + fact("node-2", false, true, false), + fact("node-3", false, true, false), + ]; + assert_eq!( + evaluate_proposal(&m, true, true, &fleet), + Some(ProposalOutcome::Refuse(RailRefusal::UnknownNode)) + ); +} + +#[test] +fn evaluate_proposal_proposes_when_eligible_and_the_rails_pass() { + let m = corroborated_contain_node_mitigation("node-1"); + let fleet = vec![ + fact("node-1", false, true, false), + fact("node-2", false, true, false), + fact("node-3", false, true, false), + fact("cp-1", true, true, false), + ]; + assert_eq!( + evaluate_proposal(&m, true, true, &fleet), + Some(ProposalOutcome::Proposed) + ); +} + +#[test] +fn evaluate_proposal_still_proposes_a_standing_protector_cordon() { + // The target is ALREADY cordoned by protector (unschedulable + owned) — the rails + // still pass (cordon_decision doesn't gate on the target's own state, only on the + // REST of the fleet). There is no "already applied, skip" branch to test here: + // `ProposalOutcome` carries no such field — proposing is idempotent by nature (it + // never writes anything), unlike the apply path this ticket deliberately does not + // have. + let m = corroborated_contain_node_mitigation("node-1"); + let fleet = vec![ + fact("node-1", false, false, true), + fact("node-2", false, true, false), + fact("node-3", false, true, false), + ]; + assert_eq!( + evaluate_proposal(&m, true, true, &fleet), + Some(ProposalOutcome::Proposed) + ); +} + +#[test] +fn evaluate_proposal_surfaces_the_cordon_rail_refusal() { + // Eligible on every upstream gate, but the target is control-plane — the rail (not + // the arming/scope/corroboration gate) is what refuses here. + let m = corroborated_contain_node_mitigation("cp-1"); + let fleet = vec![ + fact("cp-1", true, true, false), + fact("node-1", false, true, false), + ]; + assert_eq!( + evaluate_proposal(&m, true, true, &fleet), + Some(ProposalOutcome::Refuse(RailRefusal::ControlPlane)) + ); +} diff --git a/engine/src/engine/respond/actuator/tests.rs b/engine/src/engine/respond/actuator/tests.rs index df2e81b..dfb8bbf 100644 --- a/engine/src/engine/respond/actuator/tests.rs +++ b/engine/src/engine/respond/actuator/tests.rs @@ -271,12 +271,15 @@ fn decide_forbids_subtractive_rbac() { )); } -/// ADR-0040 §5/§6: `ContainNode` must never auto-apply through the generic `decide()` path -/// — there is no `node` arming rung to escalate past yet, and `is_additive_live() == false` -/// means [`decide`] forbids it structurally, before even reaching the enabled/scope checks. -/// Enabling it here (something no real config can do today — there is no operator-facing -/// `node` class name, `actuator::actions_from_name` never maps to it) proves the forbid is -/// unconditional, not just "nobody happens to enable it". +/// ADR-0040 §5/§6: `ContainNode` must never auto-apply through the GENERIC `decide()` path +/// — `is_additive_live() == false` means [`decide`] forbids it structurally, before even +/// reaching the enabled/scope checks, REGARDLESS of the arming ladder's `node` rung +/// (`arming_ladder::ArmingRung::Node`) enabling the class. `ContainNode` has NO apply path +/// at all, generic or otherwise — its own gate +/// (`node_containment::evaluate_proposal`, called from `Engine::process`) only ever +/// decides whether to SURFACE a proposal, never whether to apply one (ADR-0040 §5: +/// propose-first by construction). Enabling the class here directly proves the generic +/// forbid is unconditional, not just "nobody happens to enable it". #[test] fn decide_forbids_contain_node_even_when_the_class_would_be_enabled() { let m = mitigation( diff --git a/engine/src/engine/run_loop.rs b/engine/src/engine/run_loop.rs index 06ca239..4435306 100644 --- a/engine/src/engine/run_loop.rs +++ b/engine/src/engine/run_loop.rs @@ -403,7 +403,7 @@ pub async fn run_watch( signing_exceptions: crate::policies::signature::SigningExceptions, ) -> anyhow::Result<()> { use futures::stream::StreamExt; - use k8s_openapi::api::core::v1::{Pod, Secret, Service}; + use k8s_openapi::api::core::v1::{Node, Pod, Secret, Service}; use k8s_openapi::api::networking::v1::{Ingress, IngressClass, NetworkPolicy}; use k8s_openapi::api::rbac::v1::{ClusterRole, ClusterRoleBinding, Role, RoleBinding}; use kube::Api; @@ -449,7 +449,16 @@ pub async fn run_watch( // The offline IP→ASN dataset: read each pass to group INTERNET egress by // provider in the prompt. Shares the same swap cell we spawn the reloader on below. .with_asn(asn.clone()) - .with_break_glass(break_glass.clone()); + .with_break_glass(break_glass.clone()) + // The node-containment REVERT actuator (ADR-0040 §5): uncordon/co-resident-deny-lift + // glue for the break-glass/self-revert loop + // (`crate::engine::node_containment_revert::Engine::revert_contain_node`). There is + // deliberately no corresponding APPLY wiring here — `ContainNode` is propose-only at + // every arming rung (ADR-0040 §5's propose-first-by-construction rule), so nothing in + // `Engine::process` ever calls `NodeContainmentActuator::apply`. + .with_node_containment_actuator(Box::new( + respond::actuator::node_containment::NodeContainmentActuator::new(client.clone()), + )); // Repopulate the webhook's admission-decision ring from the durable journal on boot // so the admission-decision log isn't blank after a restart — parallel to how @@ -733,6 +742,13 @@ pub async fn run_watch( // IngressExposureAdapter's raw material. let (ingresses, ingresses_w) = reflector::store::(); let (ingress_classes, ingress_classes_w) = reflector::store::(); + // The Node fleet (ADR-0040 §3/§6) — the node-containment rails' raw material + // (`observe::adapter::node_fact::observe_node_facts`). Metadata-only in effect (only + // name/labels/spec.unschedulable/the cordon annotation are ever read from it), always + // watched regardless of posture — the always-on `nodes` read grant (see the chart's + // ClusterRole) — so the rails have real fleet data from the pass `enforceRung: node` + // is armed, not just from the pass after. + let (nodes, nodes_w) = reflector::store::(); let cfg = watcher::Config::default(); // CRITICAL: each reflector runs in its OWN task so its Store stays current no @@ -772,6 +788,7 @@ pub async fn run_watch( spawn_reflector!(rolebindings_w, RoleBinding); spawn_reflector!(clusterroles_w, ClusterRole); spawn_reflector!(clusterrolebindings_w, ClusterRoleBinding); + spawn_reflector!(nodes_w, Node); // ADR-0038: unlike every type above (always granted), the Ingress/IngressClass // RBAC can legitimately be missing (an older chart render, or the forked cluster // chart before its RBAC hand-port lands — see the ADR's rollout note). Preflight @@ -863,6 +880,8 @@ pub async fn run_watch( linkerd_servers: linkerd_servers_now, linkerd_authz_policies: linkerd_policies_now, linkerd_mtls_auths: linkerd_mtls_now, + // The Node fleet (ADR-0040 §3/§6) — the node-containment rails' raw material. + nodes: nodes.state().iter().map(|n| (**n).clone()).collect(), }; // Run the supply-chain sweep pipeline over this snapshot: observe signing posture, // opt-in Rekor reconciliation, default-on provenance observation, publish the whole-pass diff --git a/engine/src/main.rs b/engine/src/main.rs index 308a9cd..63ff52b 100644 --- a/engine/src/main.rs +++ b/engine/src/main.rs @@ -86,7 +86,10 @@ impl Posture { /// Resolve `PROTECTOR_MODE` + `PROTECTOR_ENFORCE_SCOPE_NAMESPACES` / /// `PROTECTOR_ENFORCE_SCOPE_LABELS` + `PROTECTOR_ENFORCE_RUNG`. `mode: enforce` /// with an empty scope is refused — enforcing everywhere is the footgun ADR-0021 - /// guards against (no wildcard). + /// guards against (no wildcard). `PROTECTOR_ENFORCE_RUNG=node` additionally + /// requires `mode: enforce` (with the non-empty scope the check above already + /// requires) — protector's first node-write action class must never be configured + /// without both enforcement gates armed (ADR-0040 §6). fn from_env() -> Result { let mode = env_or("PROTECTOR_MODE", "audit") .trim() @@ -106,6 +109,15 @@ impl Posture { ); } let rung = ArmingRung::from_name(&env_or("PROTECTOR_ENFORCE_RUNG", "edge-cut")); + if rung == ArmingRung::Node && !enforce { + anyhow::bail!( + "PROTECTOR_ENFORCE_RUNG=node requires PROTECTOR_MODE=enforce (with a \ + non-empty enforceScope) — refusing to start: node containment \ + (ADR-0040) is protector's first node-write action, and configuring its \ + rung without the enforce gate armed is very likely a misconfiguration, \ + not an intentional shadow rung (ADR-0035)." + ); + } Ok(Self { enforce, namespaces, @@ -501,6 +513,16 @@ mod tests { use protector::engine::respond::ProposedAction; use protector::policy::EnforceScope; use std::collections::HashSet; + use std::sync::Mutex; + + /// `cargo test` runs unit tests on multiple threads by default, but + /// `PROTECTOR_MODE`/`PROTECTOR_ENFORCE_SCOPE_NAMESPACES`/`PROTECTOR_ENFORCE_RUNG` are + /// process-global env vars — every test below that sets/clears them must hold this + /// lock for its whole body, or two such tests interleaved on different threads race + /// each other's env mutations (a `Posture::from_env()` call can observe a var another + /// concurrently-running test just set or cleared). `Mutex<()>`, not an atomic: this is + /// pure mutual exclusion, no data is actually guarded. + static ENV_TEST_LOCK: Mutex<()> = Mutex::new(()); /// Build a Posture directly, bypassing env, so the derivation is tested without /// touching process-global env. Defaults to the narrowest rung (`edge-cut`) — the @@ -630,8 +652,9 @@ mod tests { #[test] fn enforce_with_empty_scope_is_refused() { // ADR-0021: no enforce-everywhere wildcard. `mode: enforce` with an empty scope - // must fail at startup. Serial env mutation guarded like the other env tests. - // SAFETY: single-threaded within this test; vars are set + cleared here only. + // must fail at startup. Serial env mutation guarded like the other env tests + // (`ENV_TEST_LOCK`) — held for the whole test body. + let _guard = ENV_TEST_LOCK.lock().unwrap(); unsafe { std::env::set_var("PROTECTOR_MODE", "enforce"); std::env::remove_var("PROTECTOR_ENFORCE_SCOPE_NAMESPACES"); @@ -658,7 +681,7 @@ mod tests { fn enforce_rung_env_var_selects_the_ladder_position() { // ADR-0035: `PROTECTOR_ENFORCE_RUNG` unset defaults to the narrowest rung // (edge-cut-only); `quarantine` opts into the second rung. - // SAFETY: single-threaded within this test; vars are set + cleared here only. + let _guard = ENV_TEST_LOCK.lock().unwrap(); unsafe { std::env::set_var("PROTECTOR_MODE", "enforce"); std::env::set_var("PROTECTOR_ENFORCE_SCOPE_NAMESPACES", "payments"); @@ -683,4 +706,37 @@ mod tests { std::env::remove_var("PROTECTOR_ENFORCE_RUNG"); } } + + #[test] + fn enforce_rung_node_requires_enforce_mode() { + // ADR-0040 §6: `enforceRung: node` without `mode: enforce` (+ non-empty + // enforceScope) must refuse to start — protector's first node-write action + // class is never configured on a hunch. + let _guard = ENV_TEST_LOCK.lock().unwrap(); + unsafe { + std::env::remove_var("PROTECTOR_MODE"); + std::env::remove_var("PROTECTOR_ENFORCE_SCOPE_NAMESPACES"); + std::env::set_var("PROTECTOR_ENFORCE_RUNG", "node"); + } + assert!( + Posture::from_env().is_err(), + "enforceRung: node under mode: audit (the default) must be refused" + ); + + // Armed correctly (enforce + scope + node rung), it is accepted. + unsafe { + std::env::set_var("PROTECTOR_MODE", "enforce"); + std::env::set_var("PROTECTOR_ENFORCE_SCOPE_NAMESPACES", "payments"); + } + let p = Posture::from_env().expect("enforce + scope + node rung is accepted"); + assert_eq!(p.rung, ArmingRung::Node); + let (active, _scope) = p.engine_arming(); + assert!(active.is_enabled(ProposedAction::ContainNode)); + + unsafe { + std::env::remove_var("PROTECTOR_MODE"); + std::env::remove_var("PROTECTOR_ENFORCE_SCOPE_NAMESPACES"); + std::env::remove_var("PROTECTOR_ENFORCE_RUNG"); + } + } } diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 37cb9c1..e8858c2 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -365,6 +365,15 @@ rules: - apiGroups: ["aquasecurity.github.io"] resources: ["vulnerabilityreports"] verbs: ["get", "list", "watch"] + # Node fleet observation (ADR-0040 §3/§6) — metadata-only read (name, + # control-plane signal, spec.unschedulable, protector's cordon-ownership annotation); + # mirrors charts/protector/templates/clusterrole.yaml. ALWAYS ON like every other read: + # the engine's Node watch runs every pass regardless of mode, so without this grant it + # 403-floods the run loop and starves the proof passes. No `patch` — e2e never arms + # enforceRung: node (ContainNode is propose-only, ADR-0040 §5). + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: ["policy.linkerd.io"] resources: ["servers", "authorizationpolicies", "meshtlsauthentications"] verbs: ["get", "list", "watch"]$np_write