From 2132e94285d61ecfa716f959f4bb898444c8a5c7 Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Thu, 28 May 2026 13:21:37 +0200 Subject: [PATCH] Support full ExecCredentialStatus in exec-plugin providers Fixed the generic exec-plugin path to support all three credential shapes that a Kubernetes exec credential plugin can return, and updated the README to reflect the current implementation. --- README.md | 65 +++++++++++++++++++++++++++++++++---- controllers/utils.go | 67 ++++++++++++++++++++++++--------------- controllers/utils_test.go | 59 ++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f8faaca..703e9b1 100644 --- a/README.md +++ b/README.md @@ -46,15 +46,13 @@ clusterinventory-controller Sveltos (addon-controller, sveltoscluster-manager, …) ``` -## Where it stops +## Access providers -- **Access provider support**: only the `kubeconfig-secretreader` provider is supported today. This provider reads a full kubeconfig from a pre-existing Kubernetes Secret referenced inside the `client.authentication.k8s.io/exec` extension of the `ClusterProfile` status. Exec-plugin providers (where the `ClusterProfile` vends short-lived credentials via an external binary) are not yet supported; adding them requires implementing a new helper and wiring it into `getKubeconfig()` in `controllers/utils.go`. -- **Kubeconfig refresh**: the controller reconciles the kubeconfig Secret whenever the `ClusterProfile` is reconciled. It does **not** independently watch the source Secret for changes; an external actor (e.g., the cluster manager that writes the `ClusterProfile` status) is responsible for triggering a re-reconcile when credentials rotate. -- **Cluster lifecycle**: the controller does not provision, deprovision, or health-check the remote cluster. It only translates the `ClusterProfile` representation into what Sveltos needs. The `SveltosCluster` readiness check (and everything that happens after it) is entirely Sveltos's responsibility. +### kubeconfig-secretreader (built-in) -## Access provider: kubeconfig-secretreader +This is a built-in fast path that requires no external binary. The controller reads a **complete kubeconfig** from a pre-existing Kubernetes Secret and copies it into the managed kubeconfig Secret. -The `kubeconfig-secretreader` provider expects the following JSON payload embedded in the `client.authentication.k8s.io/exec` extension of the access provider entry: +The `ClusterProfile` access provider entry must be named `kubeconfig-secretreader` and carry the following JSON payload in the `client.authentication.k8s.io/exec` Cluster extension: ```json { @@ -70,6 +68,61 @@ The `kubeconfig-secretreader` provider expects the following JSON payload embedd | `key` | yes | Data key inside the Secret | | `namespace` | no | Namespace of the Secret; defaults to the `ClusterProfile` namespace | +> **Naming note**: the upstream [cluster-inventory-api](https://github.com/kubernetes-sigs/cluster-inventory-api) project also ships a binary called `kubeconfig-secretreader` that works as an exec credential plugin (returning an `ExecCredential` with token or cert/key data). The two uses of the name refer to different things: in this controller `kubeconfig-secretreader` is a built-in code path that copies a full kubeconfig, while upstream it is an exec plugin binary. A future improvement could unify them by routing `kubeconfig-secretreader` through the generic exec-plugin path described below. + +### Exec-plugin providers (via `--clusterprofile-provider-file`) + +Any access provider backed by an exec credential plugin can be enabled by passing a provider configuration file to the controller at startup: + +``` +--clusterprofile-provider-file=/etc/clusterinventory/providers.json +``` + +The controller invokes the configured binary directly at reconcile time, embeds the returned credentials into a plain kubeconfig (no exec stanza), and stores that kubeconfig in the managed Secret. This means `sveltoscluster-manager` does not need the exec binary in its own pod. + +All three credential shapes returned by `ExecCredentialStatus` are supported: + +| Returned by plugin | Written to kubeconfig | +|---|---| +| `status.token` | `user.token` | +| `status.clientCertificateData` + `status.clientKeyData` | `user.client-certificate-data` + `user.client-key-data` | +| both token and cert/key | both fields set | + +If the plugin returns an `expirationTimestamp`, the controller automatically requeues at 80 % of the remaining token lifetime (minimum 1 minute) so the Secret is refreshed before the credentials expire. For certificate-only plugins that do not set an expiry, rotation relies on the controller's `--sync-period` (default 10 minutes) or an external re-trigger of the `ClusterProfile`. + +#### Provider configuration file format + +```json +{ + "providers": [ + { + "name": "my-provider", + "execConfig": { + "apiVersion": "client.authentication.k8s.io/v1", + "command": "/usr/local/bin/my-credential-plugin", + "args": ["--cluster-name", "$(CLUSTER_NAME)"], + "env": [ + {"name": "MY_ENV", "value": "value"} + ], + "provideClusterInfo": true, + "interactiveMode": "Never" + }, + "profileSourcedCLIArgsPolicy": "Append", + "profileSourcedEnvVarsPolicy": "AppendIfNotExists" + } + ] +} +``` + +The `name` field must match the provider name in the `ClusterProfile`'s `status.accessProviders`. The `profileSourcedCLIArgsPolicy` and `profileSourcedEnvVarsPolicy` fields control whether cluster-specific arguments and environment variables embedded in the `ClusterProfile` extensions (per [KEP-5339](https://github.com/kubernetes/enhancements/issues/5339)) are merged into the plugin invocation. + +> **Operational note**: the exec plugin binary must be present and executable inside the controller pod. The controller invokes it with `KUBERNETES_EXEC_INFO` set so that plugins using `provideClusterInfo: true` receive the correct server and CA data. + +## Where it stops + +- **Kubeconfig refresh**: the controller reconciles the kubeconfig Secret whenever the `ClusterProfile` is reconciled. It does **not** independently watch the source Secret for changes (kubeconfig-secretreader path) or the token expiry beyond the scheduled requeue (exec-plugin path). An external actor (e.g., the cluster manager that writes the `ClusterProfile` status) is responsible for triggering a re-reconcile when credentials rotate out-of-band. +- **Cluster lifecycle**: the controller does not provision, deprovision, or health-check the remote cluster. It only translates the `ClusterProfile` representation into what Sveltos needs. The `SveltosCluster` readiness check (and everything that happens after it) is entirely Sveltos's responsibility. + ## Managed resources All resources created by the controller carry the label: diff --git a/controllers/utils.go b/controllers/utils.go index 4f2cb5b..19c55f8 100644 --- a/controllers/utils.go +++ b/controllers/utils.go @@ -66,7 +66,7 @@ const ( managedByValue = "clusterinventory-controller" // tokenKubeconfigCluster, tokenKubeconfigUser, and tokenKubeconfigContext are - // the fixed names used in the minimal kubeconfig built by BuildTokenKubeconfig. + // the fixed names used in the kubeconfig built by BuildKubeconfigFromExecStatus. tokenKubeconfigCluster = "cluster" tokenKubeconfigUser = "user" tokenKubeconfigContext = "context" @@ -365,27 +365,34 @@ func getKubeconfigFromExecPlugin(ctx context.Context, } logger.V(logs.LogDebug).Info("invoking exec plugin", "command", restCfg.ExecProvider.Command) - token, expiry, err := invokeExecPlugin(ctx, restCfg.ExecProvider.Command, + status, err := invokeExecPlugin(ctx, restCfg.ExecProvider.Command, restCfg.ExecProvider.Args, envVars, ap) if err != nil { return nil, nil, err } - kubeconfig, err := BuildTokenKubeconfig(ap.Cluster.Server, ap.Cluster.CertificateAuthorityData, token) + var expiry *time.Time + if status.ExpirationTimestamp != nil { + t := status.ExpirationTimestamp.Time + expiry = &t + } + + kubeconfig, err := BuildKubeconfigFromExecStatus(ap.Cluster.Server, ap.Cluster.CertificateAuthorityData, status) if err != nil { - return nil, nil, fmt.Errorf("building token kubeconfig: %w", err) + return nil, nil, fmt.Errorf("building kubeconfig from exec status: %w", err) } return kubeconfig, expiry, nil } -// invokeExecPlugin runs an exec credential plugin and returns the bearer token -// and its optional expiry time. It sets KUBERNETES_EXEC_INFO so that plugins +// invokeExecPlugin runs an exec credential plugin and returns its full +// ExecCredentialStatus. It sets KUBERNETES_EXEC_INFO so that plugins // that request cluster info (ProvideClusterInfo: true) receive the correct -// server and CA data. +// server and CA data. Returns an error if the plugin emits no status, or if +// the status contains neither a token nor client certificate data. func invokeExecPlugin(ctx context.Context, command string, args []string, envVars []string, - ap *clusterinventoryv1alpha1.AccessProvider) (string, *time.Time, error) { + ap *clusterinventoryv1alpha1.AccessProvider) (*clientauthenticationv1.ExecCredentialStatus, error) { // Build KUBERNETES_EXEC_INFO: an ExecCredential carrying the cluster // connection details so the plugin knows which cluster it is authenticating to. @@ -410,7 +417,7 @@ func invokeExecPlugin(ctx context.Context, } execInfoJSON, err := json.Marshal(execInfo) if err != nil { - return "", nil, fmt.Errorf("marshaling KUBERNETES_EXEC_INFO: %w", err) + return nil, fmt.Errorf("marshaling KUBERNETES_EXEC_INFO: %w", err) } cmd := exec.CommandContext(ctx, command, args...) @@ -419,29 +426,35 @@ func invokeExecPlugin(ctx context.Context, out, err := cmd.Output() if err != nil { - return "", nil, fmt.Errorf("exec plugin %q: %w", command, err) + return nil, fmt.Errorf("exec plugin %q: %w", command, err) } var result clientauthenticationv1.ExecCredential if err := json.Unmarshal(out, &result); err != nil { - return "", nil, fmt.Errorf("parsing exec plugin output: %w", err) + return nil, fmt.Errorf("parsing exec plugin output: %w", err) } - if result.Status == nil || result.Status.Token == "" { - return "", nil, fmt.Errorf("exec plugin %q returned no token", command) + if result.Status == nil { + return nil, fmt.Errorf("exec plugin %q returned no status", command) } - - var expiry *time.Time - if result.Status.ExpirationTimestamp != nil { - t := result.Status.ExpirationTimestamp.Time - expiry = &t + if result.Status.Token == "" && result.Status.ClientCertificateData == "" { + return nil, fmt.Errorf("exec plugin %q returned neither a token nor client certificate data", command) } - return result.Status.Token, expiry, nil + return result.Status, nil } -// BuildTokenKubeconfig constructs a minimal kubeconfig that authenticates -// with a static bearer token. The resulting YAML can be stored in a Secret -// and used by sveltoscluster-manager without any exec binary. -func BuildTokenKubeconfig(server string, caData []byte, token string) ([]byte, error) { +// BuildKubeconfigFromExecStatus constructs a minimal kubeconfig from the +// ExecCredentialStatus returned by an exec credential plugin. All credential +// fields are mapped: +// - status.Token → AuthInfo.Token +// - status.ClientCertificateData → AuthInfo.ClientCertificateData (PEM bytes) +// - status.ClientKeyData → AuthInfo.ClientKeyData (PEM bytes) +// +// Empty fields are omitted, so the result is correct for token-only, +// cert+key-only, or combined credentials. The resulting YAML can be stored in +// a Secret and used by sveltoscluster-manager without any exec binary. +func BuildKubeconfigFromExecStatus(server string, caData []byte, + status *clientauthenticationv1.ExecCredentialStatus) ([]byte, error) { + kc := clientcmdv1.Config{ APIVersion: "v1", Kind: "Config", @@ -453,8 +466,12 @@ func BuildTokenKubeconfig(server string, caData []byte, token string) ([]byte, e }, }}, AuthInfos: []clientcmdv1.NamedAuthInfo{{ - Name: tokenKubeconfigUser, - AuthInfo: clientcmdv1.AuthInfo{Token: token}, + Name: tokenKubeconfigUser, + AuthInfo: clientcmdv1.AuthInfo{ + Token: status.Token, + ClientCertificateData: []byte(status.ClientCertificateData), + ClientKeyData: []byte(status.ClientKeyData), + }, }}, Contexts: []clientcmdv1.NamedContext{{ Name: tokenKubeconfigContext, diff --git a/controllers/utils_test.go b/controllers/utils_test.go index f894c94..7f0042c 100644 --- a/controllers/utils_test.go +++ b/controllers/utils_test.go @@ -28,9 +28,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientauthenticationv1 "k8s.io/client-go/pkg/apis/clientauthentication/v1" clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1" clusterinventoryv1alpha1 "sigs.k8s.io/cluster-inventory-api/apis/v1alpha1" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/yaml" controller "github.com/projectsveltos/clusterinventory-controller/controllers" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" @@ -290,6 +292,63 @@ var _ = Describe("Utils", func() { Expect(controller.DeleteKubeconfigSecret(context.TODO(), testEnv.Client, cp, logger)).To(Succeed()) }) }) + + Context("BuildKubeconfigFromExecStatus", func() { + const ( + testServer = "https://cluster.example.com:6443" + testCertPEM = "-----BEGIN CERTIFICATE-----\nfake\n-----END CERTIFICATE-----\n" + testKeyPEM = "-----BEGIN RSA PRIVATE KEY-----\nfake\n-----END RSA PRIVATE KEY-----\n" //nolint:gosec // fake PEM, not a real key + ) + testCA := []byte("fake-ca-data") + + It("produces a token-only kubeconfig when only Token is set", func() { + status := &clientauthenticationv1.ExecCredentialStatus{Token: "my-token"} + data, err := controller.BuildKubeconfigFromExecStatus(testServer, testCA, status) + Expect(err).To(BeNil()) + + var kc clientcmdv1.Config + Expect(yaml.Unmarshal(data, &kc)).To(Succeed()) + Expect(kc.Clusters).To(HaveLen(1)) + Expect(kc.Clusters[0].Cluster.Server).To(Equal(testServer)) + Expect(kc.Clusters[0].Cluster.CertificateAuthorityData).To(Equal(testCA)) + Expect(kc.AuthInfos).To(HaveLen(1)) + Expect(kc.AuthInfos[0].AuthInfo.Token).To(Equal("my-token")) + Expect(kc.AuthInfos[0].AuthInfo.ClientCertificateData).To(BeEmpty()) + Expect(kc.AuthInfos[0].AuthInfo.ClientKeyData).To(BeEmpty()) + }) + + It("produces a cert+key kubeconfig when only certificate data is set", func() { + status := &clientauthenticationv1.ExecCredentialStatus{ + ClientCertificateData: testCertPEM, + ClientKeyData: testKeyPEM, + } + data, err := controller.BuildKubeconfigFromExecStatus(testServer, testCA, status) + Expect(err).To(BeNil()) + + var kc clientcmdv1.Config + Expect(yaml.Unmarshal(data, &kc)).To(Succeed()) + Expect(kc.AuthInfos).To(HaveLen(1)) + Expect(kc.AuthInfos[0].AuthInfo.Token).To(BeEmpty()) + Expect(kc.AuthInfos[0].AuthInfo.ClientCertificateData).To(Equal([]byte(testCertPEM))) + Expect(kc.AuthInfos[0].AuthInfo.ClientKeyData).To(Equal([]byte(testKeyPEM))) + }) + + It("includes all credential fields when token and cert+key are both set", func() { + status := &clientauthenticationv1.ExecCredentialStatus{ + Token: "combined-token", + ClientCertificateData: testCertPEM, + ClientKeyData: testKeyPEM, + } + data, err := controller.BuildKubeconfigFromExecStatus(testServer, testCA, status) + Expect(err).To(BeNil()) + + var kc clientcmdv1.Config + Expect(yaml.Unmarshal(data, &kc)).To(Succeed()) + Expect(kc.AuthInfos[0].AuthInfo.Token).To(Equal("combined-token")) + Expect(kc.AuthInfos[0].AuthInfo.ClientCertificateData).To(Equal([]byte(testCertPEM))) + Expect(kc.AuthInfos[0].AuthInfo.ClientKeyData).To(Equal([]byte(testKeyPEM))) + }) + }) }) // buildClusterProfile builds a minimal ClusterProfile for use in tests.