Remove k8s API access from NGINX pod (#4368)

Problem: The init container in the NGINX pod needed k8s API access for NGINX Plus licensing purposes. However, this data could be provided by the control plane without the init container needing the API access. For security reasons, the NGINX pod shouldn't have any access to the API.

Solution: Remove API access and provide the necessary data directly to the pod.

Author: sjberman

cmd/gateway/commands.go

@@ -13,15 +13,13 @@ import (
 	"go.uber.org/zap"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/klog/v2"
-	ctlr "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8sConfig "sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 	ctlrZap "sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/config"
-	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/licensing"
 	ngxConfig "github.com/nginx/nginx-gateway-fabric/v2/internal/controller/nginx/config"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/framework/file"
 )
@@ -667,10 +665,9 @@ func createInitializeCommand() *cobra.Command {
 				return fmt.Errorf("could not get pod UID: %w", err)
 			}
 
-			clusterCfg := ctlr.GetConfigOrDie()
-			k8sReader, err := client.New(clusterCfg, client.Options{})
+			clusterUID, err := getValueFromEnv("CLUSTER_UID")
 			if err != nil {
-				return fmt.Errorf("unable to initialize k8s client: %w", err)
+				return fmt.Errorf("could not get cluster UID: %w", err)
 			}
 
 			logger := ctlrZap.New()
@@ -684,12 +681,6 @@ func createInitializeCommand() *cobra.Command {
 			)
 			log.SetLogger(logger)
 
-			dcc := licensing.NewDeploymentContextCollector(licensing.DeploymentContextCollectorConfig{
-				K8sClientReader: k8sReader,
-				PodUID:          podUID,
-				Logger:          logger.WithName("deployCtxCollector"),
-			})
-
 			files := make([]fileToCopy, 0, len(srcFiles))
 			for i, src := range srcFiles {
 				files = append(files, fileToCopy{
@@ -702,8 +693,9 @@ func createInitializeCommand() *cobra.Command {
 				fileManager:   file.NewStdLibOSFileManager(),
 				fileGenerator: ngxConfig.NewGeneratorImpl(plus, nil, logger.WithName("generator")),
 				logger:        logger,
+				podUID:        podUID,
+				clusterUID:    clusterUID,
 				plus:          plus,
-				collector:     dcc,
 				copy:          files,
 			})
 		},

cmd/gateway/initialize.go

@@ -1,21 +1,19 @@
 package main
 
 import (
-	"context"
 	"fmt"
 	"os"
 	"path/filepath"
-	"time"
 
 	"github.com/go-logr/logr"
 
-	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/licensing"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/nginx/config"
+	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/state/dataplane"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/framework/file"
 )
 
 const (
-	collectDeployCtxTimeout = 10 * time.Second
+	integrationID = "ngf"
 )
 
 type fileToCopy struct {
@@ -24,10 +22,11 @@ type fileToCopy struct {
 }
 
 type initializeConfig struct {
-	collector     licensing.Collector
 	fileManager   file.OSFileManager
 	fileGenerator config.Generator
 	logger        logr.Logger
+	podUID        string
+	clusterUID    string
 	copy          []fileToCopy
 	plus          bool
 }
@@ -44,16 +43,12 @@ func initialize(cfg initializeConfig) error {
 		return nil
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), collectDeployCtxTimeout)
-	defer cancel()
-
-	depCtx, err := cfg.collector.Collect(ctx)
-	if err != nil {
-		cfg.logger.Error(err, "error collecting deployment context")
+	depCtx := dataplane.DeploymentContext{
+		InstallationID: &cfg.podUID,
+		ClusterID:      &cfg.clusterUID,
+		Integration:    integrationID,
 	}
 
-	cfg.logger.Info("Deployment context collected", "deployment context", depCtx)
-
 	depCtxFile, err := cfg.fileGenerator.GenerateDeploymentContext(depCtx)
 	if err != nil {
 		return fmt.Errorf("failed to generate deployment context file: %w", err)

cmd/gateway/initialize_test.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"errors"
 	"io"
 	"os"
@@ -11,7 +10,6 @@ import (
 	"github.com/go-logr/logr"
 	. "github.com/onsi/gomega"
 
-	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/licensing/licensingfakes"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/nginx/config/configfakes"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/controller/state/dataplane"
 	"github.com/nginx/nginx-gateway-fabric/v2/internal/framework/file"
@@ -91,18 +89,9 @@ func TestInitialize_Plus(t *testing.T) {
 		{
 			name:       "normal",
 			collectErr: nil,
-			depCtx: dataplane.DeploymentContext{
-				Integration:      "ngf",
-				ClusterID:        helpers.GetPointer("cluster-id"),
-				InstallationID:   helpers.GetPointer("install-id"),
-				ClusterNodeCount: helpers.GetPointer(2),
-			},
-		},
-		{
-			name:       "collecting deployment context errors",
-			collectErr: errors.New("collect error"),
 			depCtx: dataplane.DeploymentContext{
 				Integration:    "ngf",
+				ClusterID:      helpers.GetPointer("cluster-id"),
 				InstallationID: helpers.GetPointer("install-id"),
 			},
 		},
@@ -114,17 +103,11 @@ func TestInitialize_Plus(t *testing.T) {
 			g := NewWithT(t)
 
 			fakeFileMgr := &filefakes.FakeOSFileManager{}
-			fakeCollector := &licensingfakes.FakeCollector{
-				CollectStub: func(_ context.Context) (dataplane.DeploymentContext, error) {
-					return test.depCtx, test.collectErr
-				},
-			}
 			fakeGenerator := &configfakes.FakeGenerator{}
 
 			ic := initializeConfig{
 				fileManager:   fakeFileMgr,
 				logger:        logr.Discard(),
-				collector:     fakeCollector,
 				fileGenerator: fakeGenerator,
 				copy: []fileToCopy{
 					{
@@ -136,7 +119,9 @@ func TestInitialize_Plus(t *testing.T) {
 						srcFileName: "src2",
 					},
 				},
-				plus: true,
+				podUID:     "install-id",
+				clusterUID: "cluster-id",
+				plus:       true,
 			}
 
 			g.Expect(initialize(ic)).To(Succeed())
@@ -149,7 +134,6 @@ func TestInitialize_Plus(t *testing.T) {
 			// write deploy ctx
 			g.Expect(fakeGenerator.GenerateDeploymentContextCallCount()).To(Equal(1))
 			g.Expect(fakeGenerator.GenerateDeploymentContextArgsForCall(0)).To(Equal(test.depCtx))
-			g.Expect(fakeCollector.CollectCallCount()).To(Equal(1))
 			g.Expect(fakeFileMgr.WriteCallCount()).To(Equal(1))
 			g.Expect(fakeFileMgr.ChmodCallCount()).To(Equal(3))
 		})

internal/controller/provisioner/objects.go

@@ -859,13 +859,19 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec(
 	image, pullPolicy := p.buildImage(nProxyCfg)
 	tokenAudience := fmt.Sprintf("%s.%s.svc", p.cfg.GatewayPodConfig.ServiceName, p.cfg.GatewayPodConfig.Namespace)
 
+	clusterID := "unknown"
+	if p.cfg.AgentLabels != nil {
+		if val, ok := p.cfg.AgentLabels["cluster-id"]; ok {
+			clusterID = val
+		}
+	}
+
 	spec := corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:      objectMeta.Labels,
 			Annotations: podAnnotations,
 		},
 		Spec: corev1.PodSpec{
-			AutomountServiceAccountToken: helpers.GetPointer(true),
 			Containers: []corev1.Container{
 				{
 					Name:            "nginx",
@@ -926,6 +932,10 @@ func (p *NginxProvisioner) buildNginxPodTemplateSpec(
 								},
 							},
 						},
+						{
+							Name:  "CLUSTER_UID",
+							Value: clusterID,
+						},
 					},
 					VolumeMounts: []corev1.VolumeMount{
 						{MountPath: "/agent", Name: "nginx-agent-config"},