Secure CSI Driver TCP Connections via Substrate Pod Identity

cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go

@@ -113,6 +113,10 @@ func (h *Impl) MakeCert(ctx context.Context, pcr *certsv1beta1.PodCertificateReq
 			continue
 		}
 
+		if len(svc.Spec.Selector) == 0 {
+			continue
+		}
+
 		// Find the set of pods that the service selects.
 		matchedPods, err := h.kc.CoreV1().Pods(pcr.ObjectMeta.Namespace).List(ctx, metav1.ListOptions{
 			LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{MatchLabels: svc.Spec.Selector}),

hack/install-ate.sh

@@ -471,14 +471,6 @@ deploy_ate_system() {
   # schemas and RBAC (role.yaml has no other apply path).
   deploy_crds
 
-  if [[ "${SETUP_CSI:-false}" == "true" ]]; then
-    if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then
-      setup_csi
-    else
-      echo "Warning: CSI setup is only supported for Kind local installations. Skipping."
-    fi
-  fi
-
   # Enforce per-class SandboxConfig asset requirements (applied before any
   # SandboxConfig so the defaults below are validated too).
   run_kubectl apply -f manifests/ate-install/sandboxconfig-validation.yaml
@@ -514,6 +506,18 @@ deploy_ate_system() {
     run_kubectl apply -f manifests/ate-install/postgres.yaml
   fi
 
+  # CSI setup must run after podcertificate-controller is ready and trust bundles
+  # exist. The ghostunnel sidecar uses projected podCertificate and clusterTrustBundle
+  # volumes which cannot be fulfilled until podcertcontroller is actively signing,
+  # otherwise rollout of csi-hostpath-socat times out.
+  if [[ "${SETUP_CSI:-false}" == "true" ]]; then
+    if [[ "${ATE_INSTALL_KIND:-false}" == "true" ]]; then
+      setup_csi
+    else
+      echo "Warning: CSI setup is only supported for Kind local installations. Skipping."
+    fi
+  fi
+
   local manifests=""
   manifests="$(render_ate_system_manifests)"
   echo "${manifests}" | run_kubectl apply -f -
@@ -787,7 +791,7 @@ done
 # flag they configure (e.g. --benchmark-worker-count before/after
 # --deploy-benchmarks). The dispatch loop below also accepts these flags but
 # treats them as no-ops since the value is already captured here.
-SETUP_CSI=false
+SETUP_CSI="${SETUP_CSI:-false}"
 BENCHMARK_WORKER_COUNT=1
 BENCHMARK_SANDBOX_CLASS=gvisor
 prescan_args=("$@")

hack/install-demo-counter.sh

@@ -43,9 +43,16 @@ demo-counter_deploy() {
   local ext_vol_mount_cmd=("-e" "/\${EXTERNAL_VOLUME_MOUNTS}/d")
   local ext_vol_spec_cmd=("-e" "/\${EXTERNAL_VOLUMES}/d")
   if [[ "${with_external_volume}" == "true" ]]; then
+    # csi-hostpath-sc only exists when hack/setup-csi-hostpath-kind.sh has run (via SETUP_CSI=true).
+    # Otherwise fall back to the default "standard" StorageClass.
+    local storage_class="standard"
+    if [[ "${SETUP_CSI:-false}" == "true" ]]; then
+      storage_class="csi-hostpath-sc"
+    fi
+
     validate_cmd=("-e" "s|\${VALIDATE_EXISTING_FILE_PATH_ARG}|    - --validate-existing-file-path=/external-data/test.txt|g")
     ext_vol_mount_cmd=("-e" "s|\${EXTERNAL_VOLUME_MOUNTS}|    - name: external-data\n      mountPath: /external-data|g")
-    ext_vol_spec_cmd=("-e" "s|\${EXTERNAL_VOLUMES}|  - name: external-data\n    externalVolumeTemplate:\n      capacity: 1Gi\n      storageClassName: standard|g")
+    ext_vol_spec_cmd=("-e" "s|\${EXTERNAL_VOLUMES}|  - name: external-data\n    externalVolumeTemplate:\n      capacity: 1Gi\n      storageClassName: ${storage_class}|g")
   fi
 
   sed -e "s|\${BUCKET_NAME}|${BUCKET_NAME}|g" \

hack/setup-csi-hostpath-kind.sh

@@ -54,6 +54,25 @@ else
   echo "Warning: Kind node ${KIND_NODE} not running. Skipping directory cleanup."
 fi
 
+# 2 Expose CSI Controller over TCP Service.
+# This Service must be created before the driver pod deploys for the servicednssigner
+# because servicednssigner to issue a certificate.
+echo "Exposing CSI Controller over TCP Service..."
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: Service
+metadata:
+  name: csi-hostpath-controller
+  namespace: default
+spec:
+  selector:
+    app.kubernetes.io/name: csi-hostpath-socat
+  ports:
+  - port: 50051
+    targetPort: 10000
+    name: grpc
+EOF
+
 # 3. Deploy the CSI Hostpath Driver
 echo "Deploying CSI Hostpath Driver..."
 # The apply might fail at snapshotclass due to missing CRDs. We catch and ignore this.
@@ -88,23 +107,6 @@ spec:
           type: DirectoryOrCreate
 "
 
-# 5. Create the TCP Service mapping port 50051 to socat port 10000
-echo "Exposing CSI Controller over TCP Service..."
-cat <<EOF | kubectl apply -f -
-apiVersion: v1
-kind: Service
-metadata:
-  name: csi-hostpath-controller
-  namespace: default
-spec:
-  selector:
-    app.kubernetes.io/name: csi-hostpath-socat
-  ports:
-  - port: 50051
-    targetPort: 10000
-    name: grpc
-EOF
-
 # 6. Create the StorageClass
 echo "Creating csi-hostpath-sc StorageClass..."
 kubectl apply -f "${DRIVER_DIR}/examples/csi-storageclass.yaml"
@@ -125,6 +127,10 @@ spec:
   driverName: hostpath.csi.k8s.io
   controllerEndpoint: tcp://csi-hostpath-controller.default.svc.cluster.local:50051
   nodeSocketOverride: unix:///var/lib/kubelet/plugins/csi-hostpath/csi.sock
+  tls:
+    enabled: true
+    usePodIdentity: true
+    serverName: csi-hostpath-controller.default.svc
 EOF
 
 # 9. Restart atelet to recreate image cache directories if they were wiped

hack/third_party/csi-driver-host-path/deploy/csi-hostpath-testing.yaml

@@ -7,6 +7,65 @@
 #
 # This is useful for testing with csi-sanity or csc.
 
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: csi-hostpath-envoy-config
+  namespace: default
+  labels:
+    app.kubernetes.io/instance: hostpath.csi.k8s.io
+    app.kubernetes.io/part-of: csi-driver-host-path
+    app.kubernetes.io/name: csi-hostpath-socat
+    app.kubernetes.io/component: envoy
+data:
+  envoy.yaml: |
+    admin:
+      address:
+        socket_address: { address: 127.0.0.1, port_value: 15000 }
+    static_resources:
+      listeners:
+      - name: csi_tls_listener
+        address:
+          socket_address: { address: 0.0.0.0, port_value: 10000 }
+        filter_chains:
+        - transport_socket:
+            name: envoy.transport_sockets.tls
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
+              require_client_certificate: true
+              common_tls_context:
+                alpn_protocols:
+                - h2
+                tls_certificates:
+                - certificate_chain: { filename: /run/servicedns/credential-bundle.pem }
+                  private_key: { filename: /run/servicedns/credential-bundle.pem }
+                  watched_directory: { path: /run/servicedns }
+                validation_context:
+                  trusted_ca: { filename: /run/podidentity-ca/trust-bundle.pem }
+                  watched_directory: { path: /run/podidentity-ca }
+                  match_typed_subject_alt_names:
+                  - san_type: URI
+                    matcher:
+                      exact: "spiffe://cluster.local/ns/ate-system/sa/ate-api-server"
+          filters:
+          - name: envoy.filters.network.tcp_proxy
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+              stat_prefix: csi_proxy
+              cluster: csi_unix_socket
+      clusters:
+      - name: csi_unix_socket
+        connect_timeout: 0.25s
+        type: STATIC
+        load_assignment:
+          cluster_name: csi_unix_socket
+          endpoints:
+          - lb_endpoints:
+            - endpoint:
+                address:
+                  pipe:
+                    path: /csi/csi.sock
+---
 apiVersion: v1
 kind: Service
 metadata:
@@ -65,23 +124,50 @@ spec:
                   - hostpath.csi.k8s.io
             topologyKey: kubernetes.io/hostname
       containers:
-        - name: socat
-          image: registry.k8s.io/sig-storage/hostpathplugin:v1.17.1
+        - name: envoy
+          image: envoyproxy/envoy:v1.34-latest
           command:
-          - socat
+            - "/usr/local/bin/envoy"
           args:
-            - tcp-listen:10000,fork,reuseaddr
-            - unix-connect:/csi/csi.sock
+            - "-c"
+            - "/etc/envoy/envoy.yaml"
           securityContext:
             # This is necessary only for systems with SELinux, where
             # non-privileged sidecar containers cannot access unix domain socket
             # created by privileged CSI driver container.
             privileged: true
           volumeMounts:
+          - mountPath: /etc/envoy
+            name: envoy-config
           - mountPath: /csi
             name: socket-dir
+          - mountPath: /run/servicedns
+            name: servicedns-certs
+            readOnly: true
+          - mountPath: /run/podidentity-ca
+            name: podidentity-ca
+            readOnly: true
       volumes:
         - hostPath:
             path: /var/lib/kubelet/plugins/csi-hostpath
             type: DirectoryOrCreate
           name: socket-dir
+        - name: envoy-config
+          configMap:
+            name: csi-hostpath-envoy-config
+        - name: servicedns-certs
+          projected:
+            sources:
+            - podCertificate:
+                signerName: servicedns.podcert.ate.dev/identity
+                keyType: ECDSAP256
+                credentialBundlePath: credential-bundle.pem
+        - name: podidentity-ca
+          projected:
+            sources:
+            - clusterTrustBundle:
+                signerName: podidentity.podcert.ate.dev/identity
+                labelSelector:
+                  matchLabels:
+                    podcert.ate.dev/canarying: live
+                path: trust-bundle.pem

internal/volume/csi/client.go

@@ -15,12 +15,13 @@
 package csi
 
 import (
+	"crypto/tls"
 	"fmt"
-	"log/slog"
 	"net/url"
 
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/credentials/insecure"
 )
 
@@ -57,21 +58,20 @@ func parseEndpoint(endpoint string) (string, string, error) {
 
 // NewCSIClient establishes a gRPC connection to the CSI driver over UDS or TCP
 // and returns a client initialized with Identity, Controller, and Node service clients.
-func NewCSIClient(endpoint string) (*Client, error) {
-	scheme, target, err := parseEndpoint(endpoint)
+func NewCSIClient(endpoint string, tlsCfg *tls.Config) (*Client, error) {
+	_, target, err := parseEndpoint(endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	if scheme == "tcp" {
-		slog.Warn("CSI connection is unencrypted over TCP! (TODO: Implement TLS)", slog.String("endpoint", target))
+	var creds credentials.TransportCredentials
+	if tlsCfg != nil {
+		creds = credentials.NewTLS(tlsCfg)
+	} else {
+		creds = insecure.NewCredentials()
 	}
 
-	dialOpts := []grpc.DialOption{
-		grpc.WithTransportCredentials(insecure.NewCredentials()),
-	}
-
-	conn, err := grpc.NewClient(target, dialOpts...)
+	conn, err := grpc.NewClient(target, grpc.WithTransportCredentials(creds))
 	if err != nil {
 		return nil, fmt.Errorf("failed to dial CSI endpoint %q: %w", target, err)
 	}

internal/volume/csi/client_test.go

@@ -138,7 +138,7 @@ func TestNewCSIClient_TCP(t *testing.T) {
 
 	// Connect to the mock server using NewCSIClient
 	endpoint := "tcp://" + lis.Addr().String()
-	client, err := NewCSIClient(endpoint)
+	client, err := NewCSIClient(endpoint, nil)
 	if err != nil {
 		t.Fatalf("failed to create CSI client: %v", err)
 	}