diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 06d32ea..a34d212 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -55,3 +55,16 @@ jobs: run: make create-cluster fv env: FV: true + FV-PLUGIN: + runs-on: ubuntu-latest + steps: + - name: checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: 1.26.3 + - name: fv + run: make create-cluster-fv fv-exec-plugin + env: + FV: true \ No newline at end of file diff --git a/.gitignore b/.gitignore index ac16266..bca16be 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ hack/tools/bin/ # Needed by fv test/fv/workload_kubeconfig test/fv/production_kubeconfig +test/fv/exec-plugin-linux-amd64 bin/manager manager_image_patch.yaml-e diff --git a/Makefile b/Makefile index e7ff504..cccfc38 100644 --- a/Makefile +++ b/Makefile @@ -140,17 +140,28 @@ K8S_VERSION := v1.35.0 endif KIND_CONFIG ?= kind-cluster.yaml +WORKLOAD_KIND_CONFIG ?= workload-kind-cluster.yaml CONTROL_CLUSTER_NAME ?= sveltos-management +WORKLOAD_CLUSTER_NAME ?= fv-workload +SVELTOS_NETWORK_NAME ?= sveltos-kind-network TIMEOUT ?= 10m NUM_NODES ?= 1 +EXEC_PLUGIN_BINARY ?= test/fv/exec-plugin-linux-amd64 .PHONY: kind-test kind-test: test create-cluster fv ## Build docker image; start kind cluster; load docker image; run fv +.PHONY: kind-test-exec-plugin +kind-test-exec-plugin: test create-cluster-fv fv-exec-plugin ## Build images; start two kind clusters; run all FV tests including exec-plugin + .PHONY: fv fv: $(GINKGO) ## Run controller tests using an existing cluster cd test/fv; $(GINKGO) -nodes $(NUM_NODES) --label-filter='FV' --v --trace --randomize-all +.PHONY: fv-exec-plugin +fv-exec-plugin: $(GINKGO) ## Run exec-plugin FV tests against an existing two-cluster setup + cd test/fv; $(GINKGO) -nodes $(NUM_NODES) --label-filter='FV-EXECPLUGIN' --v --trace --randomize-all + .PHONY: test test: manifests generate fmt vet $(SETUP_ENVTEST) ## Run unit tests. KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test $(shell go list ./... |grep -v test/fv) $(TEST_ARGS) -coverprofile cover.out @@ -160,9 +171,38 @@ create-cluster: $(KIND) $(KUBECTL) $(KUSTOMIZE) $(ENVSUBST) ## Create a new kind $(MAKE) create-control-cluster $(MAKE) deploy-projectsveltos +.PHONY: create-cluster-fv +create-cluster-fv: $(KIND) $(KUBECTL) $(KUSTOMIZE) $(ENVSUBST) build-exec-plugin ## Create two kind clusters on a shared network and deploy the controller + docker network rm $(SVELTOS_NETWORK_NAME) 2>/dev/null || true + docker network inspect $(SVELTOS_NETWORK_NAME) > /dev/null 2>&1 || docker network create $(SVELTOS_NETWORK_NAME) + + @echo "Create the management cluster" + sed -e "s/K8S_VERSION/$(K8S_VERSION)/g" test/$(KIND_CONFIG) > test/$(KIND_CONFIG).tmp + $(KIND) create cluster --name=$(CONTROL_CLUSTER_NAME) --config test/$(KIND_CONFIG).tmp + $(MAKE) deploy-projectsveltos + + @echo "Create the workload cluster" + sed -e "s/K8S_VERSION/$(K8S_VERSION)/g" test/$(WORKLOAD_KIND_CONFIG) > test/$(WORKLOAD_KIND_CONFIG).tmp + $(KIND) create cluster --name=$(WORKLOAD_CLUSTER_NAME) --config test/$(WORKLOAD_KIND_CONFIG).tmp + + @echo "Connect both clusters to the shared docker network" + docker network connect $(SVELTOS_NETWORK_NAME) $(CONTROL_CLUSTER_NAME)-control-plane + docker network connect $(SVELTOS_NETWORK_NAME) $(WORKLOAD_CLUSTER_NAME)-control-plane + + @echo "Copy exec-plugin binary onto the management cluster node" + docker cp $(EXEC_PLUGIN_BINARY) $(CONTROL_CLUSTER_NAME)-control-plane:/usr/local/bin/test-exec-plugin + docker exec $(CONTROL_CLUSTER_NAME)-control-plane chmod 755 /usr/local/bin/test-exec-plugin + + @echo "Save workload cluster kubeconfig for FV tests" + $(KIND) get kubeconfig --name $(WORKLOAD_CLUSTER_NAME) > test/fv/workload_kubeconfig + + @echo "Switch back to management cluster context" + $(KUBECTL) config use-context kind-$(CONTROL_CLUSTER_NAME) + .PHONY: delete-cluster delete-cluster: $(KIND) ## Delete the kind cluster $(KIND) delete cluster --name $(CONTROL_CLUSTER_NAME) + $(KIND) delete cluster --name $(WORKLOAD_CLUSTER_NAME) ##@ Build @@ -170,6 +210,10 @@ delete-cluster: $(KIND) ## Delete the kind cluster build: generate fmt vet ## Build manager binary. go build -o bin/manager cmd/main.go +.PHONY: build-exec-plugin +build-exec-plugin: ## Build the FV exec-plugin binary for linux/amd64 (injected into the controller pod during exec-plugin FV tests) + GOOS=linux GOARCH=amd64 go build -o $(EXEC_PLUGIN_BINARY) ./test/fv/exec-plugin/ + .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/main.go diff --git a/controllers/utils.go b/controllers/utils.go index cc5fb13..4f2cb5b 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -224,7 +224,17 @@ func reconcileKubeconfigSecret(ctx context.Context, c client.Client, Data: map[string][]byte{kubeconfigKey: kubeconfigData}, } logger.V(logs.LogDebug).Info("creating kubeconfig secret") - return c.Create(ctx, secret) + if createErr := c.Create(ctx, secret); createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + return createErr + } + // Race: another reconcile created the secret between our Get and Create. + if err = c.Get(ctx, types.NamespacedName{Namespace: cp.Namespace, Name: secretName}, existing); err != nil { + return fmt.Errorf("failed to re-get kubeconfig secret: %w", err) + } + } else { + return nil + } } if bytes.Equal(existing.Data[kubeconfigKey], kubeconfigData) { @@ -261,7 +271,17 @@ func reconcileSveltosCluster(ctx context.Context, c client.Client, }, } logger.V(logs.LogDebug).Info("creating SveltosCluster") - return c.Create(ctx, sc) + if createErr := c.Create(ctx, sc); createErr != nil { + if !apierrors.IsAlreadyExists(createErr) { + return createErr + } + // Race: another reconcile created the SveltosCluster between our Get and Create. + if err = c.Get(ctx, types.NamespacedName{Namespace: cp.Namespace, Name: cp.Name}, existing); err != nil { + return fmt.Errorf("failed to re-get SveltosCluster: %w", err) + } + } else { + return nil + } } if existing.Spec.KubeconfigName == secretName && existing.Spec.KubeconfigKeyName == kubeconfigKey { diff --git a/go.mod b/go.mod index f74c1ae..f517a28 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.26.3 require ( github.com/TwiN/go-color v1.4.1 github.com/go-logr/logr v1.4.3 - github.com/onsi/ginkgo/v2 v2.28.3 - github.com/onsi/gomega v1.40.0 + github.com/onsi/ginkgo/v2 v2.29.0 + github.com/onsi/gomega v1.41.0 github.com/projectsveltos/libsveltos v1.10.1-0.20260521153750-a1f348424b3f github.com/spf13/pflag v1.0.10 k8s.io/api v0.36.1 diff --git a/go.sum b/go.sum index 2922c69..a6bb773 100644 --- a/go.sum +++ b/go.sum @@ -127,10 +127,10 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.28.3 h1:4JvMdwtFU0imd8fHx25OJXoDMRexnf8v5NHKYSTTji4= -github.com/onsi/ginkgo/v2 v2.28.3/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= -github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc= -github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= +github.com/onsi/ginkgo/v2 v2.29.0 h1:rfh+ZFjgJhYWRoIqVf3Uwx/W20yLrcrE2h2GmYVRaag= +github.com/onsi/ginkgo/v2 v2.29.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= +github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/test/fv/clusterprofile_test.go b/test/fv/clusterprofile_test.go index 5bd13ae..b00c667 100644 --- a/test/fv/clusterprofile_test.go +++ b/test/fv/clusterprofile_test.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" + "k8s.io/client-go/util/retry" clusterinventoryv1alpha1 "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -85,9 +86,18 @@ var _ = Describe("ClusterProfile controller", Label("FV"), func() { cp := buildFvClusterProfile(cpName, namespace) Expect(k8sClient.Create(context.TODO(), cp)).To(Succeed()) - // Status is a sub-resource; update it separately. - cp.Status = buildFvAccessProviderStatus(srcSecretName, "kubeconfig", namespace) - Expect(k8sClient.Status().Update(context.TODO(), cp)).To(Succeed()) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + currentCP := &clusterinventoryv1alpha1.ClusterProfile{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Namespace: cp.Namespace, Name: cp.Name}, + currentCP) + if err != nil { + return err + } + currentCP.Status = buildFvAccessProviderStatus(srcSecretName, "kubeconfig", namespace) + return k8sClient.Status().Update(context.TODO(), currentCP) + }) + Expect(err).To(BeNil()) expectedSecretName := cpName + "-sveltos-kubeconfig" diff --git a/test/fv/exec-plugin/main.go b/test/fv/exec-plugin/main.go new file mode 100644 index 0000000..2fcde70 --- /dev/null +++ b/test/fv/exec-plugin/main.go @@ -0,0 +1,52 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// exec-plugin is a minimal Kubernetes exec credential plugin used only in FV tests. +// It reads a bearer token from a well-known file and writes an ExecCredential JSON +// to stdout so that the clusterinventory-controller can embed it in a kubeconfig. +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +const ( + tokenFile = "/var/run/secrets/test-exec-plugin/token" //nolint: gosec // used for testing +) + +func main() { + raw, err := os.ReadFile(tokenFile) + if err != nil { + fmt.Fprintf(os.Stderr, "exec-plugin: read token: %v\n", err) + os.Exit(1) + } + + cred := map[string]interface{}{ + "apiVersion": "client.authentication.k8s.io/v1", + "kind": "ExecCredential", + "status": map[string]interface{}{ + "token": strings.TrimSpace(string(raw)), + }, + } + + if err := json.NewEncoder(os.Stdout).Encode(cred); err != nil { + fmt.Fprintf(os.Stderr, "exec-plugin: encode: %v\n", err) + os.Exit(1) + } +} diff --git a/test/fv/execplugin_test.go b/test/fv/execplugin_test.go new file mode 100644 index 0000000..a322ba2 --- /dev/null +++ b/test/fv/execplugin_test.go @@ -0,0 +1,182 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fv_test + +import ( + "context" + "encoding/json" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" + "k8s.io/client-go/util/retry" + clusterinventoryv1alpha1 "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" + "sigs.k8s.io/yaml" + + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" +) + +var _ = Describe("ClusterProfile exec-plugin provider", Label("FV-EXECPLUGIN"), func() { + var ( + namespace string + cpName string + ) + + BeforeEach(func() { + if workloadK8sClient == nil { + Skip("workload cluster not available; run with 'make create-cluster-fv'") + } + namespace = randomString() + cpName = randomString() + createNamespace(namespace) + }) + + AfterEach(func() { + cp := &clusterinventoryv1alpha1.ClusterProfile{} + if err := k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: cpName}, cp); err == nil { + cp.Finalizers = nil + _ = k8sClient.Update(context.TODO(), cp) + _ = k8sClient.Delete(context.TODO(), cp) + } + ns := &corev1.Namespace{} + if err := k8sClient.Get(context.TODO(), types.NamespacedName{Name: namespace}, ns); err == nil { + _ = k8sClient.Delete(context.TODO(), ns) + } + }) + + It("creates SveltosCluster and kubeconfig Secret using exec-plugin provider, and SveltosCluster becomes Ready", func() { + Byf("Creating ClusterProfile %s/%s with exec-plugin access provider", namespace, cpName) + cp := &clusterinventoryv1alpha1.ClusterProfile{ + ObjectMeta: metav1.ObjectMeta{ + Name: cpName, + Namespace: namespace, + }, + Spec: clusterinventoryv1alpha1.ClusterProfileSpec{ + ClusterManager: clusterinventoryv1alpha1.ClusterManager{Name: "fv-exec-plugin-manager"}, + }, + } + Expect(k8sClient.Create(context.TODO(), cp)).To(Succeed()) + + Byf("Setting ClusterProfile status with exec-plugin AccessProvider pointing at workload cluster") + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + currentCP := &clusterinventoryv1alpha1.ClusterProfile{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Namespace: cp.Namespace, Name: cp.Name}, + currentCP) + if err != nil { + return err + } + currentCP.Status = buildExecPluginAccessProviderStatus() + return k8sClient.Status().Update(context.TODO(), currentCP) + }) + Expect(err).To(BeNil()) + + expectedSecretName := cpName + "-sveltos-kubeconfig" + + Byf("Waiting for SveltosCluster %s/%s to be created", namespace, cpName) + Eventually(func() bool { + sc := &libsveltosv1beta1.SveltosCluster{} + return k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: cpName}, sc) == nil + }, timeout, pollingInterval).Should(BeTrue()) + + Byf("Verifying SveltosCluster points to the managed kubeconfig Secret") + sc := &libsveltosv1beta1.SveltosCluster{} + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: cpName}, sc)).To(Succeed()) + Expect(sc.Spec.KubeconfigName).To(Equal(expectedSecretName)) + Expect(sc.Spec.KubeconfigKeyName).To(Equal("kubeconfig")) + + Byf("Waiting for kubeconfig Secret %s/%s to be created", namespace, expectedSecretName) + Eventually(func() bool { + secret := &corev1.Secret{} + return k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: expectedSecretName}, secret) == nil + }, timeout, pollingInterval).Should(BeTrue()) + + Byf("Verifying kubeconfig Secret contains a token-based kubeconfig for the workload cluster") + secret := &corev1.Secret{} + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: expectedSecretName}, secret)).To(Succeed()) + Expect(secret.Data["kubeconfig"]).NotTo(BeEmpty()) + + // The generated kubeconfig must point at the workload cluster server, not the management cluster. + var kc clientcmdv1.Config + Expect(yaml.Unmarshal(secret.Data["kubeconfig"], &kc)).To(Succeed()) + Expect(kc.Clusters).NotTo(BeEmpty()) + Expect(kc.Clusters[0].Cluster.Server).To(Equal(workloadClusterServer)) + + Byf("Waiting for SveltosCluster %s/%s to become Ready (reachable workload cluster)", namespace, cpName) + Eventually(func() bool { + sc := &libsveltosv1beta1.SveltosCluster{} + err := k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: cpName}, sc) + if err != nil { + return false + } + return sc.Status.Ready + }, timeout, pollingInterval).Should(BeTrue()) + + Byf("Deleting ClusterProfile and verifying SveltosCluster and Secret are removed") + Expect(k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: cpName}, cp)).To(Succeed()) + Expect(k8sClient.Delete(context.TODO(), cp)).To(Succeed()) + + Eventually(func() bool { + sc := &libsveltosv1beta1.SveltosCluster{} + return apierrors.IsNotFound(k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: cpName}, sc)) + }, timeout, pollingInterval).Should(BeTrue()) + + Eventually(func() bool { + s := &corev1.Secret{} + return apierrors.IsNotFound(k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: namespace, Name: expectedSecretName}, s)) + }, timeout, pollingInterval).Should(BeTrue()) + }) +}) + +func buildExecPluginAccessProviderStatus() clusterinventoryv1alpha1.ClusterProfileStatus { + // The exec-plugin extension payload carries cluster-specific config; + // for the FV test binary it is not read, so we pass a minimal object. + extRaw, _ := json.Marshal(map[string]string{"provider": execPluginProviderName}) + + return clusterinventoryv1alpha1.ClusterProfileStatus{ + AccessProviders: []clusterinventoryv1alpha1.AccessProvider{ + { + Name: execPluginProviderName, + Cluster: clientcmdv1.Cluster{ + Server: workloadClusterServer, + CertificateAuthorityData: workloadClusterCAData, + Extensions: []clientcmdv1.NamedExtension{ + { + Name: execExtensionKey, + Extension: runtime.RawExtension{Raw: extRaw}, + }, + }, + }, + }, + }, + } +} diff --git a/test/fv/fv_suite_test.go b/test/fv/fv_suite_test.go index 52c4b94..cac3e65 100644 --- a/test/fv/fv_suite_test.go +++ b/test/fv/fv_suite_test.go @@ -18,6 +18,7 @@ package fv_test import ( "context" + "encoding/json" "fmt" "os" "testing" @@ -28,15 +29,18 @@ import ( "github.com/TwiN/go-color" ginkgotypes "github.com/onsi/ginkgo/v2/types" + appsv1 "k8s.io/api/apps/v1" authv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" "k8s.io/klog/v2" "sigs.k8s.io/cluster-api/util" @@ -52,6 +56,14 @@ var ( k8sClient client.Client scheme *runtime.Scheme testClusterKubeconfig []byte + + // workloadK8sClient is non-nil only when the workload cluster is available + // (i.e. create-cluster-fv was used). Exec-plugin FV tests skip when nil. + workloadK8sClient client.Client + workloadClusterCAData []byte + // workloadClusterServer is the API server URL reachable from inside the + // management cluster pods (Docker network hostname). + workloadClusterServer string ) const ( @@ -59,6 +71,23 @@ const ( pollingInterval = 5 * time.Second localContextName = "local" + + controllerDeploymentName = "clusterinventory-controller" + controllerDeploymentNamespace = "projectsveltos" + controllerContainerName = "manager" + + // execPluginBinaryPath is the path where 'make create-cluster-fv' places the + // exec-plugin binary on the kind node (docker cp) and where it is mounted + // into the controller pod as a hostPath volume. + execPluginBinaryPath = "/usr/local/bin/test-exec-plugin" + execPluginProviderConfigMap = "test-exec-plugin-provider" + execPluginTokenSecret = "test-exec-plugin-token" + execPluginProviderPath = "/etc/test-exec-plugin/provider.json" + execPluginTokenPath = "/var/run/secrets/test-exec-plugin/token" //nolint:gosec // not a real credential path + execPluginProviderName = "fv-exec-plugin" + + workloadClusterSAName = "clusterinventory-fv-workload" + workloadClusterSANamespace = "kube-system" ) func TestFv(t *testing.T) { @@ -98,6 +127,7 @@ var _ = BeforeSuite(func() { Expect(clientgoscheme.AddToScheme(scheme)).To(Succeed()) Expect(libsveltosv1beta1.AddToScheme(scheme)).To(Succeed()) Expect(clusterinventoryv1alpha1.AddToScheme(scheme)).To(Succeed()) + Expect(appsv1.AddToScheme(scheme)).To(Succeed()) var err error k8sClient, err = client.New(restConfig, client.Options{Scheme: scheme}) @@ -107,8 +137,316 @@ var _ = BeforeSuite(func() { // to connect back to the cluster via kubernetes.default.svc:443. testClusterKubeconfig, err = buildInClusterKubeconfig(context.TODO(), restConfig) Expect(err).NotTo(HaveOccurred()) + + // Set up exec-plugin test infrastructure if the workload cluster kubeconfig exists. + // It is written by the create-cluster-fv Makefile target. + if _, statErr := os.Stat("workload_kubeconfig"); statErr == nil { + workloadRestConfig, loadErr := clientcmd.BuildConfigFromFlags("", "workload_kubeconfig") + Expect(loadErr).NotTo(HaveOccurred()) + workloadRestConfig.QPS = 100 + workloadRestConfig.Burst = 100 + + workloadK8sClient, err = client.New(workloadRestConfig, client.Options{Scheme: scheme}) + Expect(err).NotTo(HaveOccurred()) + + workloadClusterCAData = workloadRestConfig.CAData + if len(workloadClusterCAData) == 0 && workloadRestConfig.CAFile != "" { + workloadClusterCAData, err = os.ReadFile(workloadRestConfig.CAFile) + Expect(err).NotTo(HaveOccurred()) + } + + // The workload cluster's API server is reachable from inside the management + // cluster pods via the Docker network using the container hostname. + workloadClusterServer = fmt.Sprintf("https://%s-control-plane:6443", "fv-workload") + + Expect(setupExecPluginInfrastructure(context.TODO(), restConfig, workloadRestConfig)).To(Succeed()) + } +}) + +var _ = AfterSuite(func() { + if workloadK8sClient == nil { + return + } + teardownExecPluginInfrastructure(context.TODO()) }) +// setupExecPluginInfrastructure creates all Kubernetes objects needed for exec-plugin FV tests +// and patches the controller Deployment to make the exec binary and provider file available. +// The binary itself is already on the kind node at execPluginBinaryPath, placed there by +// 'make create-cluster-fv' via docker cp. +func setupExecPluginInfrastructure(ctx context.Context, mgmtCfg, workloadCfg *rest.Config) error { + // 1. Create a service account in the workload cluster and get a token. + token, err := buildWorkloadClusterToken(ctx, workloadCfg) + if err != nil { + return fmt.Errorf("building workload cluster token: %w", err) + } + + // 2. Provider JSON that points the controller at our test exec binary. + providerJSON, err := buildProviderJSON() + if err != nil { + return fmt.Errorf("building provider JSON: %w", err) + } + + // 3. Create objects in the management cluster. + if err := createOrUpdateConfigMap(ctx, execPluginProviderConfigMap, controllerDeploymentNamespace, + map[string]string{"provider.json": string(providerJSON)}, nil); err != nil { + return fmt.Errorf("creating provider ConfigMap: %w", err) + } + if err := createOrUpdateSecret(ctx, execPluginTokenSecret, controllerDeploymentNamespace, + map[string][]byte{"token": []byte(token)}); err != nil { + return fmt.Errorf("creating token Secret: %w", err) + } + + // 4. Patch the controller Deployment to mount the binary (hostPath), provider file, and token. + if err := patchControllerDeployment(ctx); err != nil { + return fmt.Errorf("patching controller Deployment: %w", err) + } + + // 5. Wait for the Deployment rollout to complete. + mgmtClientset, err := kubernetes.NewForConfig(mgmtCfg) + if err != nil { + return fmt.Errorf("building management clientset: %w", err) + } + return waitForDeploymentRollout(ctx, mgmtClientset) +} + +// teardownExecPluginInfrastructure removes the objects created by setupExecPluginInfrastructure +// and restores the controller Deployment. +func teardownExecPluginInfrastructure(ctx context.Context) { + _ = k8sClient.Delete(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: execPluginProviderConfigMap, Namespace: controllerDeploymentNamespace}, + }) + _ = k8sClient.Delete(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: execPluginTokenSecret, Namespace: controllerDeploymentNamespace}, + }) + restoreControllerDeployment(ctx) +} + +func buildWorkloadClusterToken(ctx context.Context, cfg *rest.Config) (string, error) { + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return "", fmt.Errorf("building workload clientset: %w", err) + } + + _, err = clientset.CoreV1().ServiceAccounts(workloadClusterSANamespace).Create(ctx, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: workloadClusterSAName, Namespace: workloadClusterSANamespace}}, + metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("creating workload SA: %w", err) + } + + _, err = clientset.RbacV1().ClusterRoleBindings().Create(ctx, + &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: workloadClusterSAName + "-admin"}, + RoleRef: rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: "cluster-admin"}, + Subjects: []rbacv1.Subject{{Kind: "ServiceAccount", Name: workloadClusterSAName, Namespace: workloadClusterSANamespace}}, + }, metav1.CreateOptions{}) + if err != nil && !apierrors.IsAlreadyExists(err) { + return "", fmt.Errorf("creating workload ClusterRoleBinding: %w", err) + } + + expiry := int64(7200) + tokenResp, err := clientset.CoreV1().ServiceAccounts(workloadClusterSANamespace).CreateToken(ctx, + workloadClusterSAName, + &authv1.TokenRequest{Spec: authv1.TokenRequestSpec{ExpirationSeconds: &expiry}}, + metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("creating workload token: %w", err) + } + return tokenResp.Status.Token, nil +} + +func buildProviderJSON() ([]byte, error) { + provider := map[string]interface{}{ + "providers": []map[string]interface{}{ + { + "name": execPluginProviderName, + "execConfig": map[string]interface{}{ + "command": execPluginBinaryPath, + "apiVersion": "client.authentication.k8s.io/v1", + "args": []string{}, + "env": []interface{}{}, + "provideClusterInfo": false, + }, + }, + }, + } + return json.Marshal(provider) +} + +func createOrUpdateConfigMap(ctx context.Context, name, namespace string, + data map[string]string, binaryData map[string][]byte) error { + + cm := &corev1.ConfigMap{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, cm) + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: data, + BinaryData: binaryData, + } + return k8sClient.Create(ctx, cm) + } + if err != nil { + return err + } + cm.Data = data + cm.BinaryData = binaryData + return k8sClient.Update(ctx, cm) +} + +func createOrUpdateSecret(ctx context.Context, name, namespace string, data map[string][]byte) error { + secret := &corev1.Secret{} + err := k8sClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret) + if apierrors.IsNotFound(err) { + secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: data, + } + return k8sClient.Create(ctx, secret) + } + if err != nil { + return err + } + secret.Data = data + return k8sClient.Update(ctx, secret) +} + +func patchControllerDeployment(ctx context.Context) error { + deploy := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, + types.NamespacedName{Name: controllerDeploymentName, Namespace: controllerDeploymentNamespace}, + deploy); err != nil { + return err + } + + hostPathFile := corev1.HostPathFile + // Add volumes. + deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, + corev1.Volume{ + // Binary is placed on the kind node by 'make create-cluster-fv' via docker cp. + Name: "exec-plugin-binary", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: execPluginBinaryPath, + Type: &hostPathFile, + }, + }, + }, + corev1.Volume{ + Name: "exec-plugin-provider", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: execPluginProviderConfigMap}, + }, + }, + }, + corev1.Volume{ + Name: "exec-plugin-token", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: execPluginTokenSecret, + }, + }, + }, + ) + + // Add volume mounts and provider file arg to the manager container. + for i := range deploy.Spec.Template.Spec.Containers { + c := &deploy.Spec.Template.Spec.Containers[i] + if c.Name != controllerContainerName { + continue + } + c.VolumeMounts = append(c.VolumeMounts, + corev1.VolumeMount{ + Name: "exec-plugin-binary", + MountPath: execPluginBinaryPath, + }, + corev1.VolumeMount{ + Name: "exec-plugin-provider", + MountPath: execPluginProviderPath, + SubPath: "provider.json", + }, + corev1.VolumeMount{ + Name: "exec-plugin-token", + MountPath: execPluginTokenPath, + SubPath: "token", + }, + ) + c.Args = append(c.Args, "--clusterprofile-provider-file="+execPluginProviderPath) + } + + return k8sClient.Update(ctx, deploy) +} + +func restoreControllerDeployment(ctx context.Context) { + deploy := &appsv1.Deployment{} + if err := k8sClient.Get(ctx, + types.NamespacedName{Name: controllerDeploymentName, Namespace: controllerDeploymentNamespace}, + deploy); err != nil { + return + } + + // Remove our volumes. + testVolumeNames := map[string]bool{ + "exec-plugin-binary": true, + "exec-plugin-provider": true, + "exec-plugin-token": true, + } + filtered := deploy.Spec.Template.Spec.Volumes[:0] + for i := range deploy.Spec.Template.Spec.Volumes { + v := &deploy.Spec.Template.Spec.Volumes[i] + if !testVolumeNames[v.Name] { + filtered = append(filtered, *v) + } + } + deploy.Spec.Template.Spec.Volumes = filtered + + for i := range deploy.Spec.Template.Spec.Containers { + c := &deploy.Spec.Template.Spec.Containers[i] + if c.Name != controllerContainerName { + continue + } + mounts := c.VolumeMounts[:0] + for _, m := range c.VolumeMounts { + if !testVolumeNames[m.Name] { + mounts = append(mounts, m) + } + } + c.VolumeMounts = mounts + + args := c.Args[:0] + for _, a := range c.Args { + if a != "--clusterprofile-provider-file="+execPluginProviderPath { + args = append(args, a) + } + } + c.Args = args + } + + _ = k8sClient.Update(ctx, deploy) +} + +func waitForDeploymentRollout(ctx context.Context, clientset *kubernetes.Clientset) error { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + deploy, err := clientset.AppsV1().Deployments(controllerDeploymentNamespace). + Get(ctx, controllerDeploymentName, metav1.GetOptions{}) + if err != nil { + return err + } + if deploy.Status.UpdatedReplicas == *deploy.Spec.Replicas && + deploy.Status.ReadyReplicas == *deploy.Spec.Replicas && + deploy.Status.ObservedGeneration >= deploy.Generation { + + return nil + } + time.Sleep(pollingInterval) + } + return fmt.Errorf("timed out waiting for deployment %s/%s rollout", + controllerDeploymentNamespace, controllerDeploymentName) +} + // buildInClusterKubeconfig creates a ServiceAccount with cluster-admin rights and // returns a kubeconfig that uses https://kubernetes.default.svc:443 as the server. // This kubeconfig is valid from inside the cluster (e.g., from a controller Pod). diff --git a/test/workload-kind-cluster.yaml b/test/workload-kind-cluster.yaml new file mode 100644 index 0000000..4b277a2 --- /dev/null +++ b/test/workload-kind-cluster.yaml @@ -0,0 +1,8 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + podSubnet: "10.120.0.0/16" + serviceSubnet: "10.125.0.0/16" +nodes: +- role: control-plane + image: kindest/node:K8S_VERSION