feat: support cert

README.md

@@ -105,22 +105,9 @@ the standard `nvidia.com/gpu` resource limit, so scheduling and provisioning rea
 the same number. Do not set `nodeName` or a provider `nodeSelector` yourself — the
 placement controller owns those.
 
-> `kubectl logs` and `kubectl exec` both work on Modal, `-f`/`--tail` and `-it`
-> included: the manager serves the two kubelet routes the API server proxies.
-> `--timestamps`/`--previous`/`--since` and `-c` are ignored, and a terminal resize is
-> not forwarded. On providers that do not support them yet, both answer NotFound.
-
 ## Getting started
 
-- See [docs/deploy.md](docs/deploy.md) to install
-- See [config/samples](config/samples) for example NodePools and a runnable workload.
-- See [docs/add-a-provider.md](docs/add-a-provider.md) to add a provider backend.
-- See [docs/architecture.md](docs/architecture.md) for design details.
-- See [docs/status.md](docs/status.md) for how instance lifecycle becomes Pod and
-  NodeClaim status, per provider.
-- See [docs/kubelet-api.md](docs/kubelet-api.md) for how `kubectl logs` and `kubectl exec`
-  reach a Pod with no kubelet.
-- See [docs/metrics.md](docs/metrics.md) for what is instrumented and how to query it.
+See [docs](docs/README.md) for an overview of Nebula.
 
 ## License
 

cmd/main.go

@@ -19,6 +19,7 @@ package main
 import (
 	"context"
 	"crypto/tls"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -43,6 +44,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 
 	nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1"
 	"github.com/InftyAI/Nebula/internal/controller"
@@ -95,6 +97,7 @@ func main() {
 	var enableHTTP2 bool
 	var kubeletAddr, kubeletClientCA string
 	var costLabels string
+	var kubeletServingTLSBootstrap bool
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -129,6 +132,14 @@ func main() {
 			"by candidate shape only. Changing this changes the identity of every cost series. Values "+
 			"come from Pod labels and are NOT capped: the manager warns once if they push the cost "+
 			"metric past 5000 series, but pick keys an admission policy constrains.")
+	flag.BoolVar(&kubeletServingTLSBootstrap, "kubelet-serving-tls-bootstrap", false,
+		"Request a serving certificate for the manager Pod IP through the "+
+			"kubernetes.io/kubelet-serving CSR signer, and approve it. Required wherever the "+
+			"control plane verifies kubelet serving certificates (EKS sets "+
+			"--kubelet-certificate-authority), since the self-signed fallback fails there with "+
+			"x509: certificate signed by unknown authority. Off by default because it needs the "+
+			"RBAC to impersonate one virtual node identity — the signer signs for nobody else "+
+			"(see addServingCertificateBootstrap).")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -308,7 +319,7 @@ func main() {
 	// The kubelet endpoint for `kubectl logs` — one listener shared by every provider's
 	// node, hence built here rather than in setupVirtualNodes. Nil is supported: the
 	// nodes then advertise no address, and logs report NotFound.
-	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA)
+	kubeletSrv := setupKubeletServer(mgr, kubeletAddr, kubeletClientCA, kubeletServingTLSBootstrap)
 
 	// Controller and webhook registration is deferred until the cert exists, so it
 	// runs in a goroutine: the cert cannot be minted until the manager is STARTED
@@ -465,7 +476,8 @@ func setupControllers(mgr ctrl.Manager, blocklist *failover.Blocklist, kubeletSr
 // what the API server dials and nothing substitutes for it: a Service would balance to
 // a non-leader replica, which holds no tracked Pods. Either way only logs degrade, so
 // it is logged loudly and the manager carries on.
-func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletServer {
+
+func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string, servingTLSBootstrap bool) *vnode.KubeletServer {
 	if addr == "" {
 		setupLog.Info("kubelet API disabled by configuration; `kubectl logs` will not work for Nebula pods")
 		return nil
@@ -487,10 +499,77 @@ func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string) *vnode.KubeletS
 		setupLog.Error(err, "unable to add the kubelet API to the manager")
 		return nil
 	}
-	setupLog.Info("kubelet API enabled", "addr", addr, "advertisedIP", podIP, "clientCertRequired", clientCA != "")
+	if servingTLSBootstrap {
+		if err := addServingCertificateBootstrap(mgr, srv, podIP); err != nil {
+			setupLog.Error(err, "kubelet serving certificate bootstrap is off; "+
+				"the endpoint keeps its self-signed certificate, which a control plane that sets "+
+				"--kubelet-certificate-authority (EKS) rejects")
+		}
+	}
+	setupLog.Info("kubelet API enabled",
+		"addr", addr,
+		"advertisedIP", podIP,
+		"clientCertRequired", clientCA != "",
+		"servingTLSBootstrap", servingTLSBootstrap)
 	return srv
 }
 
+// Detached from the doc comment below: controller-gen ignores an rbac marker inside a
+// declaration's doc. A resourceName containing a colon must be QUOTED, or the marker fails to
+// parse and takes every other rbac rule in the package with it.
+//
+// Keep the CSR names in step with vnode.ServingCSRName, and the users with the providers that
+// can register. Only `create` cannot be scoped by name.
+// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,verbs=create
+// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,resourceNames={nebula-kubelet-serving-nebula-aws,nebula-kubelet-serving-nebula-modal,nebula-kubelet-serving-nebula-fake},verbs=delete;get
+// +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/approval,resourceNames={nebula-kubelet-serving-nebula-aws,nebula-kubelet-serving-nebula-modal,nebula-kubelet-serving-nebula-fake},verbs=update
+// +kubebuilder:rbac:groups=certificates.k8s.io,resources=signers,resourceNames=kubernetes.io/kubelet-serving,verbs=approve
+// +kubebuilder:rbac:groups="",resources=users,resourceNames={"system:node:nebula-aws","system:node:nebula-modal","system:node:nebula-fake"},verbs=impersonate
+// +kubebuilder:rbac:groups="",resources=groups,resourceNames="system:nodes",verbs=impersonate
+
+// addServingCertificateBootstrap requests a trusted serving certificate for the kubelet
+// endpoint, IMPERSONATING a virtual node to do it — see vnode.NodeIdentity for why the signer
+// requires that.
+//
+// WHICH node is arbitrary: they all advertise this one endpoint at this Pod's IP, and the API
+// server verifies against the address it dialed. First name, for determinism.
+func addServingCertificateBootstrap(mgr ctrl.Manager, srv *vnode.KubeletServer, podIP string) error {
+	names := provider.Names()
+	if len(names) == 0 {
+		return errors.New("no provider registered, so there is no virtual node to request a certificate as")
+	}
+	nodeName := vnode.NodeName(names[0])
+
+	// Impersonation is confined to this client; NewKubeletServingCertificateBootstrapper takes
+	// the unimpersonated one too, and says which call needs which.
+	cfg := rest.CopyConfig(mgr.GetConfig())
+	cfg.Impersonate = rest.ImpersonationConfig{
+		UserName: vnode.NodeIdentity(nodeName),
+		Groups:   []string{"system:nodes"},
+	}
+	nodeClient, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return err
+	}
+	ownClient, err := kubernetes.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		return err
+	}
+	bootstrapper, err := vnode.NewKubeletServingCertificateBootstrapper(
+		nodeClient,
+		ownClient,
+		srv,
+		podIP,
+		nodeName,
+		managerNamespace(),
+		os.Getenv("POD_NAME"),
+	)
+	if err != nil {
+		return err
+	}
+	return mgr.Add(bootstrapper)
+}
+
 // setupVirtualNodes adds a vnode.Runner to the manager for every registered
 // provider. The Runner needs a typed clientset (the virtual kubelet's node/pod
 // controllers use client-go directly, not the controller-runtime client), built

config/manager/manager.yaml

@@ -63,6 +63,7 @@ spec:
         args:
           - --leader-elect
           - --health-probe-bind-address=:8081
+          # - --kubelet-serving-tls-bootstrap=true
         image: controller:latest
         name: manager
         imagePullPolicy: IfNotPresent
@@ -95,6 +96,13 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.podIP
+          # Recorded as annotations on the kubelet-serving CSR, so an operator looking
+          # at a stuck request can tell which manager Pod asked for it. The CSR's name
+          # comes from the node, not from here (see vnode.ServingCSRName).
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
         envFrom:
           # Provider credentials live in a per-provider Secret, one secretRef per
           # provider — NOT a single shared secret. This matches the "creds-absent →
@@ -127,11 +135,11 @@ spec:
         # kubelet). Declaring it is documentation and NetworkPolicy surface; the
         # listener binds either way.
         #
-        # It serves TLS with a self-signed cert but does NOT verify client certs by
-        # default, because which CA signs the API server's kubelet client cert is not
-        # portable — requiring it would break logs on managed control planes. So
-        # anything able to reach this port can read any Nebula pod's logs: restrict it
-        # with a NetworkPolicy, or set --kubelet-client-ca to require mTLS.
+        # It starts with a self-signed cert, then requests and self-approves a
+        # kubelet-serving cert for POD_IP. Managed control planes that verify kubelet
+        # certificates use the signed cert once it is issued. Client certs are still not
+        # verified by default: restrict this port with a NetworkPolicy, or set
+        # --kubelet-client-ca to require mTLS.
         - name: kubelet-api
           containerPort: 10250
           protocol: TCP

config/rbac/role.yaml

@@ -20,6 +20,14 @@ rules:
   verbs:
   - create
   - patch
+- apiGroups:
+  - ""
+  resourceNames:
+  - system:nodes
+  resources:
+  - groups
+  verbs:
+  - impersonate
 - apiGroups:
   - ""
   resources:
@@ -52,6 +60,16 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - system:node:nebula-aws
+  - system:node:nebula-fake
+  - system:node:nebula-modal
+  resources:
+  - users
+  verbs:
+  - impersonate
 - apiGroups:
   - admissionregistration.k8s.io
   resources:
@@ -61,6 +79,41 @@ rules:
   - list
   - update
   - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - create
+- apiGroups:
+  - certificates.k8s.io
+  resourceNames:
+  - nebula-kubelet-serving-nebula-aws
+  - nebula-kubelet-serving-nebula-fake
+  - nebula-kubelet-serving-nebula-modal
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - delete
+  - get
+- apiGroups:
+  - certificates.k8s.io
+  resourceNames:
+  - nebula-kubelet-serving-nebula-aws
+  - nebula-kubelet-serving-nebula-fake
+  - nebula-kubelet-serving-nebula-modal
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  resources:
+  - signers
+  verbs:
+  - approve
 - apiGroups:
   - coordination.k8s.io
   resources:

docs/README.md

@@ -0,0 +1,8 @@
+# Nebula docs
+
+- [architecture.md](architecture.md) — the design: scheduling gates, virtual nodes, NodePool and NodeClaim.
+- [deploy.md](deploy.md) — installing the manager, provider credentials, manager flags.
+- [status.md](status.md) — how an instance's lifecycle becomes Pod and NodeClaim status.
+- [kubelet-api.md](kubelet-api.md) — `kubectl logs` and `kubectl exec` against a Pod with no kubelet.
+- [metrics.md](metrics.md) — what is instrumented, and how to query it.
+- [add-a-provider.md](add-a-provider.md) — adding a provider backend.

docs/deploy.md

@@ -130,13 +130,39 @@ Manager flags worth knowing (edit `config/manager/manager.yaml` `args`):
 | Flag | Default | Meaning |
 |---|---|---|
 | `--kubelet-bind-address` | `:10250` | Where the kubelet log endpoint listens — the address the API server proxies `kubectl logs` to. Set it empty to disable the endpoint, which disables logs and nothing else. |
+| `--kubelet-serving-tls-bootstrap` | `false`, but `manager.yaml` ships `true` | Request a certificate for the advertised Pod IP from the `kubernetes.io/kubelet-serving` signer, and approve it. **Required on EKS and any control plane that sets `--kubelet-certificate-authority`**, where the self-signed fallback makes `kubectl exec` fail with `x509: certificate signed by unknown authority`. The flag's own default is off because the feature needs the `impersonate` grant on `users`/`groups` in `config/rbac/role.yaml`; the shipped manifest turns it on. |
 | `--kubelet-client-ca` | *(empty)* | PEM bundle of CAs whose client certificates are accepted on that port. **Empty means client certificates are not verified**, so anything able to reach port 10250 can read the logs of any Pod on Nebula's virtual nodes. Set it to your API server's kubelet client CA to require mTLS, or keep the port closed with a NetworkPolicy. The default is open because which CA signs that client cert is not portable — kubeadm uses the cluster CA, EKS/GKE their own — so requiring it by default would break logs on managed control planes. |
 
 The endpoint needs `POD_IP` (projected via `fieldRef` in `config/manager/manager.yaml`)
 because virtual nodes advertise the leader's Pod IP, not a Service. Running the manager
 off-cluster leaves it unset, and logs degrade to unsupported. See
 [kubelet-api.md](kubelet-api.md).
 
+With `--kubelet-serving-tls-bootstrap` enabled the manager submits and approves the request
+itself, once at startup and again at each renewal, so there is no manual step. To check it
+landed:
+
+```bash
+# Approved,Issued is the healthy state. "Approved" alone means the signer refused the
+# request, which is what happens when the identity it was submitted under is not a node.
+kubectl get csr -l app.kubernetes.io/component=kubelet-serving-certificate
+
+kubectl -n nebula-system logs deploy/nebula-controller-manager \
+  | grep 'installed trusted kubelet serving certificate'
+```
+
+Two identities are involved, and the split is not cosmetic. The CSR is **created** while
+impersonating `system:node:nebula-<provider>`, because the signer signs for nobody else;
+everything else — the stale delete, the polling, the approval — goes out as the manager's
+ServiceAccount, because a node identity may create and get its own CSRs and nothing more. A
+single-identity version fails on the delete and never creates a CSR at all.
+
+The request is named `nebula-kubelet-serving-<node>`, one per virtual node for the life of the
+cluster, which is what lets `config/rbac/role.yaml` scope delete, get and approval to those
+names by `resourceNames`. Only `create` is cluster-wide. An external approver, if you run one,
+should match on that node identity, the `system:nodes` organization, and the current manager
+Pod IP as the sole IP SAN.
+
 ---
 
 ## Modal Environments
@@ -206,6 +232,10 @@ kubectl -n nebula-system logs deploy/nebula-controller-manager | grep -i provide
 # Virtual nodes exist, one per registered provider.
 kubectl get nodes -l nebula.inftyai.com/provider
 
+# Kubelet serving CSR reached Approved,Issued — only with --kubelet-serving-tls-bootstrap.
+kubectl get csr \
+  -l app.kubernetes.io/name=nebula,app.kubernetes.io/component=kubelet-serving-certificate
+
 # Webhook TLS is wired: the caBundle matches the serving cert Secret.
 diff <(kubectl get secret nebula-webhook-server-cert -n nebula-system -o jsonpath='{.data.tls\.crt}') \
      <(kubectl get mutatingwebhookconfiguration nebula-mutating-webhook-configuration \

docs/kubelet-api.md

@@ -5,6 +5,7 @@ Nebula Pod. It is worth spelling out how, because none of the usual kubelet mach
 present.
 
 - [The transport](#the-transport)
+- [The serving certificate](#the-serving-certificate)
 - [The provider seam](#the-provider-seam)
 - [What logs honour, and the one heuristic](#what-logs-honour-and-the-one-heuristic)
 - [Containers are not addressable](#containers-are-not-addressable)
@@ -23,16 +24,49 @@ Pod IP and that port. Consequences worth knowing:
 - The endpoint is **leader-scoped and dialed by Pod IP**, not through a Service. The
   tracked Pods live in one process's memory, so a Service balancing across replicas
   would send requests to a replica that answers `NotFound`.
-- It serves TLS with a self-signed, in-memory certificate — what the API server
-  expects of a kubelet, which does not verify it unless
-  `--kubelet-certificate-authority` is set. Client certificates are **not** verified
-  by default, because which CA signs the API server's kubelet client cert is not
-  portable across distributions. Anything that can reach the port can therefore read the
-  logs of, and **run commands in**, any Pod on these virtual nodes, with no RBAC check:
-  keep it closed with a NetworkPolicy, or pass `--kubelet-client-ca` to require mTLS.
+- It serves TLS on a certificate signed by the cluster CA, falling back to a self-signed
+  one until that is issued — see [The serving certificate](#the-serving-certificate).
+- Client certificates are **not** verified by default, because which CA signs the API
+  server's kubelet client cert is not portable across distributions. Serving-certificate
+  bootstrap secures the opposite direction and does not change that. Anything that can
+  reach the port can therefore read logs and **run commands in** any Pod on these virtual
+  nodes with no RBAC check: keep it closed with a NetworkPolicy, or pass
+  `--kubelet-client-ca` to require mTLS.
 - No POD_IP (running the manager off-cluster) means no endpoint. Logs and exec degrade
   to unsupported; nothing else is affected.
 
+## The serving certificate
+
+A managed control plane sets `--kubelet-certificate-authority` (EKS does) and rejects a
+self-signed kubelet certificate, so `kubectl exec` fails with `x509: certificate signed by
+unknown authority`. `--kubelet-serving-tls-bootstrap` — off in the flag, **on in
+`config/manager/manager.yaml`** — requests a real one from the `kubernetes.io/kubelet-serving`
+signer, and swaps it in without a restart. Until it lands, the endpoint keeps the self-signed
+fallback, so nothing depends on the request succeeding.
+
+The mechanics that are easy to get wrong:
+
+- **The requester is a node, and it is checked.** The CSR is created while impersonating
+  `system:node:nebula-<provider>`, with that same name as its CN. The signer signs for the node
+  that asks and for nobody else, and it reports a mismatch **nowhere** — the CSR sits
+  `Approved` with no certificate. `Approved,Issued` is the only healthy state.
+- **Two identities, not one.** Only the create is impersonated. The manager's own
+  ServiceAccount does the delete, the polling and the approval, because a node identity may
+  create and get its own CSRs and nothing more. Requester and approver differing is ordinary:
+  the signer cares only who asked.
+- **One certificate covers every virtual node.** All of them advertise the same address — this
+  Pod's IP — and the API server verifies against the address it dialed, not the node name. So
+  one request, under the first registered provider's node name, serves the whole set.
+- **One CSR per node, named `nebula-kubelet-serving-<node>`.** Stable rather than generated, so
+  `config/rbac/role.yaml` can scope delete, get and approval to those names by `resourceNames`;
+  only `create` is cluster-wide.
+- **Renewal is unattended.** 30 days requested, re-requested 24h before expiry with a fresh
+  ECDSA key that never leaves memory. A failed attempt retains the current certificate and
+  retries in 30s; a failed *approval* is retried in place, so a transient API error costs a poll
+  interval rather than the day the CSR cleaner takes to clear an unapproved request.
+
+Inspection commands are in [deploy.md](deploy.md#configuration).
+
 ## The provider seam
 
 Both are optional: a provider opts in by implementing `provider.LogStreamer` and