Add support for kubelet cert

cmd/main.go

@@ -84,6 +84,7 @@ func main() {
 	var secureMetrics bool
 	var enableHTTP2 bool
 	var kubeletAddr, kubeletClientCA string
+	var kubeletServingCSR bool
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -110,6 +111,13 @@ func main() {
 			"serves TLS without client verification, because which CA signs the API server's kubelet "+
 			"client certificate is not portable across distributions; restrict the port with a "+
 			"NetworkPolicy, or set this to your API server's kubelet client CA.")
+	flag.BoolVar(&kubeletServingCSR, "kubelet-serving-csr", false,
+		"Request the kubelet API's serving certificate from the cluster's kubernetes.io/kubelet-serving "+
+			"signer instead of self-signing it. Set this if `kubectl logs` fails with "+
+			"\"certificate signed by unknown authority\", which means the API server runs with "+
+			"--kubelet-certificate-authority. Needs the RBAC in config/rbac/kubelet_serving_role.yaml "+
+			"(it self-approves its own CSR); without it, and on a control plane whose signer is "+
+			"disabled, it logs and falls back to self-signed.")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -270,7 +278,7 @@ func main() {
 	// The kubelet endpoint for `kubectl logs` — one listener shared by every provider's
 	// node, hence built here rather than in setupVirtualNodes. Nil is supported: the
 	// nodes then advertise no address, and logs report NotFound.
-	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA)
+	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA, kubeletServingCSR)
 
 	// Controller and webhook registration is deferred until the cert exists, so it
 	// runs in a goroutine: the cert cannot be minted until the manager is STARTED
@@ -417,7 +425,7 @@ func setupControllers(mgr ctrl.Manager, blocklist *failover.Blocklist, kubeletSr
 // what the API server dials and nothing substitutes for it: a Service would balance to
 // a non-leader replica, which holds no tracked Pods. Either way only logs degrade, so
 // it is logged loudly and the manager carries on.
-func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletServer {
+func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string, servingCSR bool) *vnode.KubeletServer {
 	if addr == "" {
 		setupLog.Info("kubelet API disabled by configuration; `kubectl logs` will not work for Nebula pods")
 		return nil
@@ -435,11 +443,27 @@ func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletS
 		setupLog.Error(err, "unable to set up the kubelet API; `kubectl logs` will not work for Nebula pods")
 		return nil
 	}
+	if servingCSR {
+		// Only reachable because registerProviders ran first; without a provider there is no
+		// virtual node, and nothing to serve logs for. The name is only the CSR's subject —
+		// one issued cert covers every node here, since it is the IP SAN that matters.
+		names := provider.Names()
+		clientset, err := kubernetes.NewForConfig(mgr.GetConfig())
+		switch {
+		case err != nil:
+			setupLog.Error(err, "unable to build a clientset for the kubelet serving CSR; self-signing instead")
+		case len(names) == 0:
+			setupLog.Info("no provider registered; skipping the kubelet serving CSR")
+		default:
+			srv.EnableServingCSR(clientset, vnode.NodeName(names[0]))
+		}
+	}
 	if err := mgr.Add(srv); err != nil {
 		setupLog.Error(err, "unable to add the kubelet API to the manager")
 		return nil
 	}
-	setupLog.Info("kubelet API enabled", "addr", addr, "advertisedIP", podIP, "clientCertRequired", clientCA != "")
+	setupLog.Info("kubelet API enabled", "addr", addr, "advertisedIP", podIP,
+		"clientCertRequired", clientCA != "", "servingCSR", servingCSR)
 	return srv
 }
 

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: inftyai/nebula-controller
-  newTag: latest
+  newTag: 0819-02

config/manager/manager.yaml

@@ -63,6 +63,15 @@ spec:
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          # Uncomment together with kubelet_serving_role{,_binding}.yaml in
+          # config/rbac/kustomization.yaml — the flag asks, the RBAC permits, and both are
+          # needed. Required wherever the API server runs with
+          # --kubelet-certificate-authority, which rejects this endpoint's self-signed cert
+          # and so breaks `kubectl logs` and `kubectl exec` (kind and EKS both set it).
+          # Harmless but pointless elsewhere — and it does NOT fix EKS, whose signer ignores
+          # a CSR from a ServiceAccount and leaves it approved but unsigned. Verified on
+          # kind. See docs/kubelet-api.md.
+          #- --kubelet-serving-csr
         image: controller:latest
         name: manager
         imagePullPolicy: IfNotPresent
@@ -128,6 +137,11 @@ spec:
         # portable — requiring it would break logs on managed control planes. So
         # anything able to reach this port can read any Nebula pod's logs: restrict it
         # with a NetworkPolicy, or set --kubelet-client-ca to require mTLS.
+        #
+        # The self-signed cert is only accepted by an API server that does not verify it.
+        # If logs fail with "certificate signed by unknown authority", the control plane
+        # sets --kubelet-certificate-authority: uncomment --kubelet-serving-csr in the args
+        # above, and the two kubelet_serving_role files in config/rbac/kustomization.yaml.
         - name: kubelet-api
           containerPort: 10250
           protocol: TCP

config/rbac/kubelet_serving_role.yaml

@@ -0,0 +1,46 @@
+# Opt-in RBAC for --kubelet-serving-csr. Not generated from kubebuilder markers and not
+# applied by default, on purpose: `approve` on the kubelet-serving signer lets the holder
+# obtain a serving certificate for ANY node's kubelet endpoint, so granting it has to be a
+# deliberate act.
+#
+# Needed only where the API server runs with --kubelet-certificate-authority, which makes
+# it verify the certificate the kubelet API presents. Without these rules the manager logs
+# the failure and serves a self-signed certificate, which such an API server rejects — so
+# `kubectl logs` and `kubectl exec` on Nebula pods keep failing.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/name: nebula
+    app.kubernetes.io/managed-by: kustomize
+  name: kubelet-serving-role
+rules:
+# No update: the request is immutable once created, and a stale one (previous pod IP) is
+# replaced by delete-then-create.
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - get
+  - create
+  - delete
+# kube-controller-manager auto-approves node CLIENT certificates only, never serving ones,
+# because an approver cannot verify that a requester owns the SANs it asks for. This is the
+# manager asserting that about itself.
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+# Approval is scoped to one signer: it authorizes nothing about client certificates, which
+# are identities the API server would authenticate.
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  verbs:
+  - approve

config/rbac/kubelet_serving_role_binding.yaml

@@ -0,0 +1,17 @@
+# Binds the opt-in kubelet-serving permissions to the manager. Applied together with
+# kubelet_serving_role.yaml — see the rationale there.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/name: nebula
+    app.kubernetes.io/managed-by: kustomize
+  name: kubelet-serving-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-serving-role
+subjects:
+- kind: ServiceAccount
+  name: controller-manager
+  namespace: system

config/rbac/kustomization.yaml

@@ -9,6 +9,13 @@ resources:
 - role_binding.yaml
 - leader_election_role.yaml
 - leader_election_role_binding.yaml
+# Uncomment these two together with the #- --kubelet-serving-csr line in
+# config/manager/manager.yaml — the flag asks and this permits, so neither half works
+# alone. Needed when the API server runs with --kubelet-certificate-authority. Left out by
+# default because approving on the kubelet-serving signer is a privileged grant — see
+# kubelet_serving_role.yaml.
+#- kubelet_serving_role.yaml
+#- kubelet_serving_role_binding.yaml
 # The following RBAC configurations are used to protect
 # the metrics endpoint with authn/authz. These configurations
 # ensure that only authorized users and service accounts

config/samples/deployment.yaml

@@ -29,7 +29,7 @@ metadata:
   labels:
     app.kubernetes.io/managed-by: nebula
 spec:
-  replicas: 30
+  replicas: 3
   selector:
     matchLabels:
       app: gpu-workload-sample

docs/kubelet-api.md

@@ -5,6 +5,7 @@ Nebula Pod. It is worth spelling out how, because none of the usual kubelet mach
 present.
 
 - [The transport](#the-transport)
+- [When the API server verifies the certificate](#when-the-api-server-verifies-the-certificate)
 - [The provider seam](#the-provider-seam)
 - [What logs honour, and the one heuristic](#what-logs-honour-and-the-one-heuristic)
 - [Containers are not addressable](#containers-are-not-addressable)
@@ -25,14 +26,146 @@ Pod IP and that port. Consequences worth knowing:
   would send requests to a replica that answers `NotFound`.
 - It serves TLS with a self-signed, in-memory certificate — what the API server
   expects of a kubelet, which does not verify it unless
-  `--kubelet-certificate-authority` is set. Client certificates are **not** verified
-  by default, because which CA signs the API server's kubelet client cert is not
-  portable across distributions. Anything that can reach the port can therefore read the
-  logs of, and **run commands in**, any Pod on these virtual nodes, with no RBAC check:
-  keep it closed with a NetworkPolicy, or pass `--kubelet-client-ca` to require mTLS.
+  `--kubelet-certificate-authority` is set. Where it is set, that certificate is
+  rejected and both commands fail until the CSR path below is enabled.
+- Client certificates are **not** verified by default, because which CA signs the API
+  server's kubelet client cert is not portable across distributions. Anything that can
+  reach the port can therefore read the logs of, and **run commands in**, any Pod on
+  these virtual nodes, with no RBAC check: keep it closed with a NetworkPolicy, or pass
+  `--kubelet-client-ca` to require mTLS.
 - No POD_IP (running the manager off-cluster) means no endpoint. Logs and exec degrade
   to unsupported; nothing else is affected.
 
+## When the API server verifies the certificate
+
+An API server started with `--kubelet-certificate-authority` checks the certificate the
+kubelet API presents, and a self-signed one fails:
+
+```
+Error from server: Get "https://10.244.0.6:10250/containerLogs/default/my-pod/workload":
+tls: failed to verify certificate: x509: certificate signed by unknown authority
+```
+
+Publishing our own CA somewhere the API server would trust it is not an option: `Node`
+has no `caBundle` field, unlike `APIService` and the webhook configurations. The only
+certificate that API server accepts is one from the cluster's own
+`kubernetes.io/kubelet-serving` signer, so the manager asks for one
+(`pkg/vnode/servingcert.go`). Two switches, both off by default.
+
+### Enabling it
+
+**1. Grant the RBAC.** Uncomment both resources in `config/rbac/kustomization.yaml`:
+
+```yaml
+- kubelet_serving_role.yaml
+- kubelet_serving_role_binding.yaml
+```
+
+**2. Pass the flag.** Uncomment the last line of the manager's `args` in
+`config/manager/manager.yaml`:
+
+```yaml
+        args:
+          - --leader-elect
+          - --health-probe-bind-address=:8081
+          - --kubelet-serving-csr
+```
+
+Both halves are needed and neither is enough: the flag asks, the RBAC permits.
+
+**3. Apply**, with `make deploy IMG=<your image>`, then confirm the request was issued and
+that both commands work:
+
+```console
+$ kubectl get csr | grep nebula-kubelet-serving
+nebula-kubelet-serving-nebula-aws   8s   kubernetes.io/kubelet-serving   system:serviceaccount:nebula-system:nebula-controller-manager   <none>   Approved,Issued
+
+$ kubectl logs <a-nebula-pod>
+$ kubectl exec <a-nebula-pod> -- sh -c 'echo ok'
+```
+
+One certificate covers every virtual node this manager hosts: the API server verifies the
+SAN, which is the manager's Pod IP, and not the `system:node:<name>` subject the signer
+insists on — so the CSR is named after whichever provider registered first, and that is
+not a mistake.
+
+To enable it on an already-running deployment instead of redeploying, apply the same two
+RBAC files and append the flag with
+`kubectl -n nebula-system patch deploy nebula-controller-manager --type=json -p
+'[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--kubelet-serving-csr"}]'`.
+The restart changes the manager's Pod IP, and with it the address the nodes advertise and
+the SAN in the new request — all three move together, so nothing needs coordinating.
+
+### Why the RBAC is separate
+
+Because the manager **self-approves its own request**:
+kube-controller-manager auto-approves node *client* certificates only, never serving
+ones, since an approver cannot verify that a requester owns the SANs it asks for. So the
+grant includes `approve` on that signer, which is the power to obtain a serving
+certificate for any node's kubelet endpoint — a deliberate act, hence opt-in. What it is
+not is a node identity: the issued certificate carries the `serverAuth` usage only, so
+despite its `system:node:<name>` subject it cannot authenticate *as* a node.
+
+### Renewal, and what failure looks like
+
+The certificate is renewed at two thirds of its life, because the signer's
+`--cluster-signing-duration` is the cluster's to choose and a short one would otherwise
+lose logs mid-run.
+
+Nothing here is fatal: every failure degrades to the self-signed certificate and retries
+every 10 minutes, so granting the RBAC late takes effect without a restart. The two things
+to read are `signerIssued` in the `serving kubelet api` startup line and the CSR's own
+condition, which name the cause between them:
+
+| symptom | cause |
+|---|---|
+| no CSR at all, `signerIssued=false` | the flag is not set |
+| `create`/`approve ... is forbidden` in the log | the RBAC is not applied |
+| CSR stays `Approved` and never reaches `Issued` | the cluster's signer will not sign it — either disabled, or (EKS) restricted to node identities; not fixable from here |
+| `signerIssued=true` but still `x509` | the API server's `--kubelet-certificate-authority` is a different CA than the signer's |
+
+### Which clusters need it, and where it works
+
+kind and EKS both set `--kubelet-certificate-authority`, so on either one the self-signed
+certificate is rejected and `kubectl logs` / `kubectl exec` fail on Nebula pods with the
+x509 error above — on a connection the control plane otherwise reached fine.
+
+Needing it and being able to use it are different things, though:
+
+| cluster | outcome |
+|---|---|
+| kind | works — CSR reaches `Approved,Issued` and both commands then succeed |
+| EKS | **does not work**: the CSR stays `Approved` and is never signed |
+
+EKS's signer only signs requests whose *requester* is a node identity. Ours is the
+manager's ServiceAccount, so the request is ignored — no certificate, and no `Failed`
+condition to say why. Measured on EKS 1.35 by submitting the identical CSR bytes with the
+identical approval under two identities: signed as `system:node:<name>` (group
+`system:nodes`), unsigned as
+`system:serviceaccount:nebula-system:nebula-controller-manager`. Self-approval is not the
+obstacle — an admin-approved CSR from the ServiceAccount is ignored just the same.
+
+So do not infer that the signer will sign for us from the fact that it signs for real
+kubelets: `kubectl logs` on a pod on a real node succeeds on EKS (its nodes run
+`serverTLSBootstrap: true`) while our request goes unsigned. The only proof is our own CSR
+reaching `Issued`. An empty `kubectl get csr` proves nothing either way — issued CSRs are
+garbage-collected about an hour later.
+
+### On kind
+
+kind has a second, unrelated fault worth recognising: it sets the kubelet's
+`serverTLSBootstrap: true` but
+ships nothing that approves the resulting CSRs. So the *real* kubelet never gets a serving
+certificate either, and `kubectl logs` fails for every pod in the cluster — with `remote
+error: tls: internal error` rather than the x509 error above, since a kubelet with no
+certificate cannot complete the handshake at all. Nebula's pods are unaffected by that (it
+approves its own), but `kubectl certificate approve` on the pending `system:node:` requests
+is what fixes the rest.
+
+kind also signs for 15 minutes, which makes the renewal loop load-bearing here rather than
+decorative: a `kubectl logs` that worked at startup and fails twenty minutes later is a
+rotation failure, not an issuance one.
+
 ## The provider seam
 
 Both are optional: a provider opts in by implementing `provider.LogStreamer` and